diff --git a/.github/workflows/publish_docs.yml b/.github/workflows/publish_docs.yml deleted file mode 100644 index 4b39bc884139..000000000000 --- a/.github/workflows/publish_docs.yml +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Publish Documentation - -on: - workflow_dispatch: - schedule: - # Every working day at 12pm UTC - - cron: '0 12 * * 1-5' - -jobs: - linux: - runs-on: ubuntu-18.04 - env: - IREE_DOC_BUILD_DIR: build-docs - steps: - - name: Setup Bazel - uses: abhinavsingh/setup-bazel@v3 - with: - version: 3.7.1 - - name: Checking out repository - uses: actions/checkout@v2 - with: - token: ${{ secrets.GITHUB_WRITE_ACCESS_TOKEN }} - - name: Initializing submodules - run: ./scripts/git/submodule_versions.py init - - name: Installing Ninja build - uses: seanmiddleditch/gha-setup-ninja@v3 - - name: Building documentation - run: | - ./build_tools/cmake/build_docs.sh - # Patch the MarkDown files with front matter for rendering - ./scripts/prepare_doc_publication.py ${IREE_DOC_BUILD_DIR}/doc - - name: Fetching gh-pages branch - run: | - git fetch origin gh-pages - - name: Updating gh-pages branch - run: | - git checkout -f gh-pages - cp -rf ${IREE_DOC_BUILD_DIR}/doc/* docs/ - # The index.md will be the landing page. It should be placed in the - # top directory. - mv -f docs/index.md . - git add docs/ index.md - echo "has_diff=false" >> $GITHUB_ENV - git diff --cached --exit-code || echo "has_diff=true" >> $GITHUB_ENV - - name: Committing updates - if: env.has_diff == 'true' - run: | - git config --local user.email "iree-github-actions-bot@google.com" - git config --local user.name "Doc Publish Action" - git commit -am "Automatically update GitHub Pages" - - name: Pushing changes - if: env.has_diff == 'true' - uses: ad-m/github-push-action@v0.5.0 - with: - github_token: ${{ secrets.GITHUB_WRITE_ACCESS_TOKEN }} - branch: gh-pages diff --git a/.gitignore b/.gitignore index edf39b67843a..4f52c9395349 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,7 @@ gradlew gradlew.bat gradle/ local.properties + +# Generated documentation files +mkdocs/site/ +docs/website/site/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0f7897dc0aa5..d84f56316c90 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -58,7 +58,7 @@ for them (e.g. because they don't have write access). ## Peculiarities Our documentation on -[repository management](https://github.com/google/iree/blob/main/docs/developing_iree/repository_management.md) +[repository management](https://github.com/google/iree/blob/main/docs/developers/developing_iree/repository_management.md) has more information on some of the oddities in our repository setup and workflows. For the most part, these should be transparent to normal developer workflows. diff --git a/README.md b/README.md index 1f16c4914b60..38a6c30a52d2 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ hardware/API-specific binaries like SPIR-V. The architecture of IREE is best illustrated by the following picture: -![IREE Architecture](./docs/iree_architecture.svg) +![IREE Architecture](./docs/developers/iree_architecture.svg) Being compilation-based means IREE does not have a traditional runtime that dispatches "ops" to their fat kernel implementations. What IREE provides is a @@ -111,12 +111,12 @@ IREE aims to ## Roadmap and Milestones IREE is in the early stages of development and not yet ready for broad adoption. -Check out the [long-term design roadmap](./docs/design_roadmap.md) to get -a sense of where we're headed. +Check out the [long-term design roadmap](./docs/developers/design_roadmap.md) to +get a sense of where we're headed. We plan on a quarterly basis using [OKRs](https://en.wikipedia.org/wiki/OKR). -Review our latest [objectives](./docs/objectives.md) to get a sense of what -we're up to in the near term. +Review our latest [objectives](./docs/developers/objectives.md) to get a sense +of what we're up to in the near term. We use [GitHub Projects](https://github.com/google/iree/projects) to track progress on IREE components and specific efforts. We use diff --git a/build_tools/cmake/build_docs.sh b/build_tools/cmake/build_docs.sh deleted file mode 100755 index 380edb74bb55..000000000000 --- a/build_tools/cmake/build_docs.sh +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Build the IREE documentation with CMake. Designed for CI, but can be run -# manually. This uses previously cached build results and does not clear the -# build directory. - -set -x -set -e - -ROOT_DIR=$(git rev-parse --show-toplevel) -BUILD_DIR=${BUILD_DIR:-"build-docs"} - -CMAKE_BIN=${CMAKE_BIN:-$(which cmake)} - -"$CMAKE_BIN" --version -ninja --version - -cd ${ROOT_DIR?} -if [ -d ${BUILD_DIR} ] -then - echo "Build directory already exists. Will use cached results there." -else - echo "Build directory does not already exist. Creating a new one." - mkdir ${BUILD_DIR} -fi -cd ${BUILD_DIR} - -"$CMAKE_BIN" .. -DCMAKE_BUILD_TYPE=Release \ - -DIREE_BUILD_COMPILER=ON \ - -DIREE_BUILD_TESTS=ON \ - -DIREE_BUILD_SAMPLES=OFF \ - -DIREE_BUILD_DOCS=ON \ - -DIREE_BUILD_PYTHON_BINDINGS=OFF \ - -G Ninja -# Generate docs and also build iree-opt for generating more docs :) -ninja iree-doc iree_tools_iree-opt - -cd ${ROOT_DIR} -mkdir -p ${BUILD_DIR}/doc/ -# Copy docs in source tree over -cp README.md ${BUILD_DIR}/doc/index.md -cp -rf docs/* ${BUILD_DIR}/doc/ -cp integrations/tensorflow/e2e/README.md \ - ${BUILD_DIR}/doc/developing_iree/tensorflow_integrations.md - -# Update op_coverage.md -scripts/update_op_coverage.py ${BUILD_DIR} - -# Generate IR conversion flow examples -build_one_example() { - example=$1 - target=$2 - - example_name=$(basename ${example%.mlir}) - filepath=${BUILD_DIR}/doc/ir_examples/${example_name}-${target}.md - title="${example_name} on ${target}" - - echo -e "# ${title}\n" > ${filepath} - - preamble=" -This is an IR conversion example auto-generated by running \`iree-opt\` -over [${example_name}](https://github.com/google/iree/tree/main/${example}). - -### The command -" - echo -e "${preamble}" >> ${filepath} - echo -e -n '```shell' >> ${filepath} - - cmd=" -iree-opt -iree-transformation-pipeline \\ - -iree-hal-target-backends=${target} \\ - --print-ir-after-all \\ - -mlir-disable-threading \\ - -mlir-elide-elementsattrs-if-larger=8 \\ - ${example} -" - echo -e "${cmd}" >> ${filepath} - echo -e '```' >> ${filepath} - - echo -e "### Input IR\n" >> ${filepath} - echo -e '{% raw %}\n```' >> ${filepath} - cat ${example} >> ${filepath} - - tmpfile=$(mktemp) - ${BUILD_DIR}/iree/tools/iree-opt \ - -iree-transformation-pipeline \ - -iree-hal-target-backends=${target} \ - -print-ir-after-all \ - -mlir-disable-threading \ - -mlir-elide-elementsattrs-if-larger=8 \ - ${example} 1>/dev/null 2>${tmpfile} - - # Reformat the IR dump into markdown. - # * Add "###"" subheader sections for each IR snippet - # * Wrap each IR snippet in {% raw %} {% endraw %} to block jekyll from - # running liquid template replacement within the code blocks - # - # Before: - # // *** IR Dump After {PASS_NAME_A} *** - # module { foo } - # - # // *** IR Dump After {PASS_NAME_B} *** - # module { bar } - # - # After: - # ### IR Dump After {PASS_NAME_A} - # - # {% raw %} - # ``` - # module { foo } - # ``` - # {% endraw %} - # - # ### IR Dump After {PASS_NAME_B} - # - # {% raw %} - # ``` - # module { bar } - # ``` - # {% endraw %} - - # Turn pass comment into headers and insert raw/endraw liquid template tags - sed 's!^// \*\*\* IR Dump After \(.*\) \*\*\*$!```\n{% endraw %}\n\n### IR Dump After \1\n\n{% raw %}\n```!' \ - ${tmpfile} >> ${filepath} - # Remove extra empty lines - sed -i '/^$/N;/^\n$/D' ${filepath} - # TODO(scotttodd): Remove extra newlines between } and ``` at the end of IR blocks - echo -e '```\n{% endraw %}' >> ${filepath} -} - -mkdir -p ${BUILD_DIR}/doc/ir_examples - -build_one_example iree/samples/ops/mhlo-dot.mlir dylib-llvm-aot -build_one_example iree/samples/ops/mhlo-dot.mlir vmla -build_one_example iree/samples/ops/mhlo-dot.mlir vulkan-spirv -build_one_example iree/samples/ops/dynamic-mhlo-dot.mlir vmla diff --git a/docs/developers/README.md b/docs/developers/README.md new file mode 100644 index 000000000000..a2b959960fb8 --- /dev/null +++ b/docs/developers/README.md @@ -0,0 +1,4 @@ +# IREE Developer Documentation + +Project documentation, developer guides, and other pages not published on +IREE's user-facing website. diff --git a/docs/debugging/releases.md b/docs/developers/debugging/releases.md similarity index 100% rename from docs/debugging/releases.md rename to docs/developers/debugging/releases.md diff --git a/docs/design_docs/codegen_passes.md b/docs/developers/design_docs/codegen_passes.md similarity index 100% rename from docs/design_docs/codegen_passes.md rename to docs/developers/design_docs/codegen_passes.md diff --git a/docs/design_docs/cuda_backend.md b/docs/developers/design_docs/cuda_backend.md similarity index 100% rename from docs/design_docs/cuda_backend.md rename to docs/developers/design_docs/cuda_backend.md diff --git a/docs/design_docs/dynamic_shapes.md b/docs/developers/design_docs/dynamic_shapes.md similarity index 100% rename from docs/design_docs/dynamic_shapes.md rename to docs/developers/design_docs/dynamic_shapes.md diff --git a/docs/design_docs/function_abi.md b/docs/developers/design_docs/function_abi.md similarity index 100% rename from docs/design_docs/function_abi.md rename to docs/developers/design_docs/function_abi.md diff --git a/docs/design_docs/hal_driver_features.md b/docs/developers/design_docs/hal_driver_features.md similarity index 100% rename from docs/design_docs/hal_driver_features.md rename to docs/developers/design_docs/hal_driver_features.md diff --git a/docs/design_docs/hlo_to_linalg.png b/docs/developers/design_docs/hlo_to_linalg.png old mode 100755 new mode 100644 similarity index 100% rename from docs/design_docs/hlo_to_linalg.png rename to docs/developers/design_docs/hlo_to_linalg.png diff --git a/docs/design_docs/linalg_to_spirv.png b/docs/developers/design_docs/linalg_to_spirv.png old mode 100755 new mode 100644 similarity index 100% rename from docs/design_docs/linalg_to_spirv.png rename to docs/developers/design_docs/linalg_to_spirv.png diff --git a/docs/design_docs/simple_ir_walkthrough.md b/docs/developers/design_docs/simple_ir_walkthrough.md similarity index 100% rename from docs/design_docs/simple_ir_walkthrough.md rename to docs/developers/design_docs/simple_ir_walkthrough.md diff --git a/docs/design_roadmap.md b/docs/developers/design_roadmap.md similarity index 100% rename from docs/design_roadmap.md rename to docs/developers/design_roadmap.md diff --git a/docs/developing_iree/benchmarking.md b/docs/developers/developing_iree/benchmarking.md similarity index 100% rename from docs/developing_iree/benchmarking.md rename to docs/developers/developing_iree/benchmarking.md diff --git a/docs/developing_iree/ccache.md b/docs/developers/developing_iree/ccache.md similarity index 100% rename from docs/developing_iree/ccache.md rename to docs/developers/developing_iree/ccache.md diff --git a/docs/developing_iree/contributor_tips.md b/docs/developers/developing_iree/contributor_tips.md similarity index 100% rename from docs/developing_iree/contributor_tips.md rename to docs/developers/developing_iree/contributor_tips.md diff --git a/docs/developing_iree/developer_overview.md b/docs/developers/developing_iree/developer_overview.md similarity index 100% rename from docs/developing_iree/developer_overview.md rename to docs/developers/developing_iree/developer_overview.md diff --git a/docs/developing_iree/e2e_benchmarking.md b/docs/developers/developing_iree/e2e_benchmarking.md similarity index 100% rename from docs/developing_iree/e2e_benchmarking.md rename to docs/developers/developing_iree/e2e_benchmarking.md diff --git a/docs/developing_iree/profiling.md b/docs/developers/developing_iree/profiling.md similarity index 100% rename from docs/developing_iree/profiling.md rename to docs/developers/developing_iree/profiling.md diff --git a/docs/developing_iree/profiling_cpu_events.md b/docs/developers/developing_iree/profiling_cpu_events.md similarity index 100% rename from docs/developing_iree/profiling_cpu_events.md rename to docs/developers/developing_iree/profiling_cpu_events.md diff --git a/docs/developing_iree/profiling_vulkan_gpu.md b/docs/developers/developing_iree/profiling_vulkan_gpu.md similarity index 100% rename from docs/developing_iree/profiling_vulkan_gpu.md rename to docs/developers/developing_iree/profiling_vulkan_gpu.md diff --git a/docs/developing_iree/profiling_with_tracy.md b/docs/developers/developing_iree/profiling_with_tracy.md similarity index 100% rename from docs/developing_iree/profiling_with_tracy.md rename to docs/developers/developing_iree/profiling_with_tracy.md diff --git a/docs/developing_iree/repository_management.md b/docs/developers/developing_iree/repository_management.md similarity index 100% rename from docs/developing_iree/repository_management.md rename to docs/developers/developing_iree/repository_management.md diff --git a/docs/developing_iree/sanitizers.md b/docs/developers/developing_iree/sanitizers.md similarity index 100% rename from docs/developing_iree/sanitizers.md rename to docs/developers/developing_iree/sanitizers.md diff --git a/docs/developing_iree/testing_guide.md b/docs/developers/developing_iree/testing_guide.md similarity index 100% rename from docs/developing_iree/testing_guide.md rename to docs/developers/developing_iree/testing_guide.md diff --git a/docs/get_started/cmake_options_and_variables.md b/docs/developers/get_started/cmake_options_and_variables.md similarity index 100% rename from docs/get_started/cmake_options_and_variables.md rename to docs/developers/get_started/cmake_options_and_variables.md diff --git a/docs/get_started/generic_vulkan_env_setup.md b/docs/developers/get_started/generic_vulkan_env_setup.md similarity index 100% rename from docs/get_started/generic_vulkan_env_setup.md rename to docs/developers/get_started/generic_vulkan_env_setup.md diff --git a/docs/get_started/getting_started_android_cmake.md b/docs/developers/get_started/getting_started_android_cmake.md similarity index 100% rename from docs/get_started/getting_started_android_cmake.md rename to docs/developers/get_started/getting_started_android_cmake.md diff --git a/docs/get_started/getting_started_linux_bazel.md b/docs/developers/get_started/getting_started_linux_bazel.md similarity index 100% rename from docs/get_started/getting_started_linux_bazel.md rename to docs/developers/get_started/getting_started_linux_bazel.md diff --git a/docs/get_started/getting_started_linux_cmake.md b/docs/developers/get_started/getting_started_linux_cmake.md similarity index 100% rename from docs/get_started/getting_started_linux_cmake.md rename to docs/developers/get_started/getting_started_linux_cmake.md diff --git a/docs/get_started/getting_started_linux_vulkan.md b/docs/developers/get_started/getting_started_linux_vulkan.md similarity index 100% rename from docs/get_started/getting_started_linux_vulkan.md rename to docs/developers/get_started/getting_started_linux_vulkan.md diff --git a/docs/get_started/getting_started_macos_bazel.md b/docs/developers/get_started/getting_started_macos_bazel.md similarity index 100% rename from docs/get_started/getting_started_macos_bazel.md rename to docs/developers/get_started/getting_started_macos_bazel.md diff --git a/docs/get_started/getting_started_macos_cmake.md b/docs/developers/get_started/getting_started_macos_cmake.md similarity index 100% rename from docs/get_started/getting_started_macos_cmake.md rename to docs/developers/get_started/getting_started_macos_cmake.md diff --git a/docs/get_started/getting_started_python.md b/docs/developers/get_started/getting_started_python.md similarity index 100% rename from docs/get_started/getting_started_python.md rename to docs/developers/get_started/getting_started_python.md diff --git a/docs/get_started/getting_started_riscv_cmake.md b/docs/developers/get_started/getting_started_riscv_cmake.md similarity index 100% rename from docs/get_started/getting_started_riscv_cmake.md rename to docs/developers/get_started/getting_started_riscv_cmake.md diff --git a/docs/get_started/getting_started_tensorflow.md b/docs/developers/get_started/getting_started_tensorflow.md similarity index 100% rename from docs/get_started/getting_started_tensorflow.md rename to docs/developers/get_started/getting_started_tensorflow.md diff --git a/docs/get_started/getting_started_windows_bazel.md b/docs/developers/get_started/getting_started_windows_bazel.md similarity index 100% rename from docs/get_started/getting_started_windows_bazel.md rename to docs/developers/get_started/getting_started_windows_bazel.md diff --git a/docs/get_started/getting_started_windows_cmake.md b/docs/developers/get_started/getting_started_windows_cmake.md similarity index 100% rename from docs/get_started/getting_started_windows_cmake.md rename to docs/developers/get_started/getting_started_windows_cmake.md diff --git a/docs/get_started/getting_started_windows_vulkan.md b/docs/developers/get_started/getting_started_windows_vulkan.md similarity index 100% rename from docs/get_started/getting_started_windows_vulkan.md rename to docs/developers/get_started/getting_started_windows_vulkan.md diff --git a/docs/iree_architecture.svg b/docs/developers/iree_architecture.svg similarity index 100% rename from docs/iree_architecture.svg rename to docs/developers/iree_architecture.svg diff --git a/docs/iree_community.md b/docs/developers/iree_community.md similarity index 100% rename from docs/iree_community.md rename to docs/developers/iree_community.md diff --git a/docs/objectives.md b/docs/developers/objectives.md similarity index 100% rename from docs/objectives.md rename to docs/developers/objectives.md diff --git a/docs/tensorflow_coverage/language_and_speech_coverage.md b/docs/developers/tensorflow_coverage/language_and_speech_coverage.md similarity index 100% rename from docs/tensorflow_coverage/language_and_speech_coverage.md rename to docs/developers/tensorflow_coverage/language_and_speech_coverage.md diff --git a/docs/tensorflow_coverage/tf_base_coverage.md b/docs/developers/tensorflow_coverage/tf_base_coverage.md similarity index 100% rename from docs/tensorflow_coverage/tf_base_coverage.md rename to docs/developers/tensorflow_coverage/tf_base_coverage.md diff --git a/docs/tensorflow_coverage/tf_keras_coverage.md b/docs/developers/tensorflow_coverage/tf_keras_coverage.md similarity index 100% rename from docs/tensorflow_coverage/tf_keras_coverage.md rename to docs/developers/tensorflow_coverage/tf_keras_coverage.md diff --git a/docs/tensorflow_coverage/vision_coverage.md b/docs/developers/tensorflow_coverage/vision_coverage.md similarity index 100% rename from docs/tensorflow_coverage/vision_coverage.md rename to docs/developers/tensorflow_coverage/vision_coverage.md diff --git a/iree/base/signature_parser.h b/iree/base/signature_parser.h index 1b9ea5e61d21..d943ca235021 100644 --- a/iree/base/signature_parser.h +++ b/iree/base/signature_parser.h @@ -298,8 +298,8 @@ class RawSignatureParser { // Parser for signatures generated by SipSignatureMangler. // This uses a Visitor interface to walk either input or result structs. // -// Mangler for the 'sip' ABI. See docs/design_docs/function_abi.md in the -// documentation. +// Mangler for the 'sip' ABI. See docs/developers/design_docs/function_abi.md in +// the documentation. class SipSignatureParser { public: enum class StructType { diff --git a/iree/compiler/Bindings/SIP/Utils/SignatureBuilder.h b/iree/compiler/Bindings/SIP/Utils/SignatureBuilder.h index 661d00519d75..12104b6028f2 100644 --- a/iree/compiler/Bindings/SIP/Utils/SignatureBuilder.h +++ b/iree/compiler/Bindings/SIP/Utils/SignatureBuilder.h @@ -81,7 +81,7 @@ class SignatureBuilder { // ----------------------------------------------------------------------------- // Mangles raw function signatures. -// See docs/design_docs/function_abi.md. +// See docs/developers/design_docs/function_abi.md. class RawSignatureMangler { public: static SignatureBuilder ToFunctionSignature(const SignatureBuilder& inputs, @@ -122,8 +122,8 @@ class RawSignatureMangler { // Mangles function signatures according to the Sip (Structured Index Path) V1 // scheme. // -// Mangler for the 'sip' ABI. See docs/design_docs/function_abi.md in the -// documentation. +// Mangler for the 'sip' ABI. See docs/developers/design_docs/function_abi.md +// in the documentation. class SipSignatureMangler { public: enum class IndexMode { diff --git a/iree/compiler/Bindings/SIP/Utils/SignatureParser.h b/iree/compiler/Bindings/SIP/Utils/SignatureParser.h index 25dc1fbb207a..a335f720b430 100644 --- a/iree/compiler/Bindings/SIP/Utils/SignatureParser.h +++ b/iree/compiler/Bindings/SIP/Utils/SignatureParser.h @@ -253,8 +253,8 @@ class RawSignatureParser { // Parser for signatures generated by SipSignatureMangler. // This uses a Visitor interface to walk either input or result structs. // -// Mangler for the 'sip' ABI. See docs/design_docs/function_abi.md in the -// documentation. +// Mangler for the 'sip' ABI. See docs/developers/design_docs/function_abi.md +// in the documentation. class SipSignatureParser { public: enum class StructType { diff --git a/iree/schemas/bytecode_module_def.fbs b/iree/schemas/bytecode_module_def.fbs index ee117c7eff41..29ba947d8f69 100644 --- a/iree/schemas/bytecode_module_def.fbs +++ b/iree/schemas/bytecode_module_def.fbs @@ -50,7 +50,7 @@ table FunctionSignatureDef { // Function level reflection attributes. // These are typically used to communicate additional ABI metadata needed // for dynamic invocation and host language mapping. - // See: docs/design_docs/function_abi.md + // See: docs/developers/design_docs/function_abi.md reflection_attrs:[ReflectionAttrDef]; } diff --git a/iree/vm/module.h b/iree/vm/module.h index 87ff642a0fa6..8c6b2ca29d31 100644 --- a/iree/vm/module.h +++ b/iree/vm/module.h @@ -334,7 +334,7 @@ typedef struct iree_vm_module { // attributes. // Returns IREE_STATUS_NOT_FOUND if index >= the number of attributes for // the function. - // See: docs/design_docs/function_abi.md + // See: docs/developers/design_docs/function_abi.md iree_status_t(IREE_API_PTR* get_function_reflection_attr)( void* self, iree_vm_function_linkage_t linkage, iree_host_size_t ordinal, iree_host_size_t index, iree_string_view_t* key, @@ -394,7 +394,8 @@ iree_vm_function_signature(const iree_vm_function_t* function); // Returns the empty string if the reflection data in general or the specific // key is not found. // -// See: docs/design_docs/function_abi.md for documentation on the ABI. +// See: docs/developers/design_docs/function_abi.md for documentation on the +// ABI. IREE_API_EXPORT iree_string_view_t iree_vm_function_reflection_attr( const iree_vm_function_t* function, iree_string_view_t key); @@ -405,7 +406,7 @@ IREE_API_EXPORT iree_string_view_t iree_vm_function_reflection_attr( // attributes. // Returns IREE_STATUS_NOT_FOUND if index >= the number of attributes for // the function. -// See: docs/design_docs/function_abi.md +// See: docs/developers/design_docs/function_abi.md IREE_API_EXPORT iree_status_t iree_vm_get_function_reflection_attr( iree_vm_function_t function, iree_host_size_t index, iree_string_view_t* key, iree_string_view_t* value); diff --git a/scripts/prepare_doc_publication.py b/scripts/prepare_doc_publication.py deleted file mode 100755 index 43b250e9815a..000000000000 --- a/scripts/prepare_doc_publication.py +++ /dev/null @@ -1,251 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Prepares MarkDown documentation publication. - -The in-tree and auto-generated MarkDown documentation lacks necessary metadata -(i.e., front matter) for specifying the layout, title, and others that are -required by Jekyll for rendering the HTML. This script patches MarkDown -files with that. -""" - -import argparse -import os - - -def parse_arguments(): - """Parses command-line options.""" - parser = argparse.ArgumentParser( - description='Processes MarkDown files for publication') - parser.add_argument('base_dir', - metavar='PATH', - type=str, - help='Base documentation directory.') - - parsed_args = parser.parse_args() - if not os.path.isdir(parsed_args.base_dir): - raise parser.error('expected path to a directory') - - return parsed_args - - -# A dictionary containing source file to doc title mappings. -# -# By default the generated doc will have a title matching the MarkDown H1 -# header. This dictionary will overrule that default behavior. -DOC_TITLE_DICT = { - 'index.md': 'Home', - 'getting_started_linux_bazel.md': 'Linux with Bazel', - 'getting_started_linux_cmake.md': 'Linux with CMake', - 'getting_started_linux_vulkan.md': 'Linux with Vulkan', - 'getting_started_windows_bazel.md': 'Windows with Bazel', - 'getting_started_windows_cmake.md': 'Windows with CMake', - 'getting_started_windows_vulkan.md': 'Windows with Vulkan', - 'getting_started_macos_bazel.md': 'macOS with Bazel', - 'getting_started_macos_cmake.md': 'macOS with CMake', - 'getting_started_android_cmake.md': 'Android with CMake', - 'generic_vulkan_env_setup.md': 'Generic Vulkan Setup', - 'getting_started_python.md': 'Python', - 'getting_started_tensorflow.md': 'TensorFlow', - 'design_roadmap.md': 'Long-term Design Roadmap', - 'objectives.md': 'Short-term Objectives', - 'tensorflow_integrations.md': 'TensorFlow Integrations', - 'e2e_benchmarking.md': 'Benchmarking TensorFlow with IREE and TFLite', - 'profiling_cpu_events.md': 'Profiling CPU events', - 'profiling_with_tracy.md': 'Profiling with Tracy', - 'profiling_vulkan_gpu.md': 'Profiling Vulkan GPU', -} - -# A dictionary containing source file to permanent link mappings. -# -# By default a source file will have a permanent URL link following its -# filename. For example, if we have docs/Foo/Bar.md, then the permanent link -# for it would be https://google.github.io/iree/Foo/Bar. This dictionary -# allows one to override the permanent link if necessary. -PERMALINK_DICT = { - 'index.md': '/', -} - -# A dictionary containing source file to navigation order mappings. -# -# By default the rendered docs will be sort alphabetically and listed on -# the left panel of https://google.github.io/iree website. This allows one -# to specify an order for a specific doc. -NAVI_ORDER_DICT = { - # Top level entries - 'index.md': 1, - # 'Getting Started' is 2. - # 'Developing IREE' is 3. - 'design_roadmap.md': 4, - 'objectives.md': 5, - 'xla_op_coverage.md': 6, - # 'Tensorflow Coverage' is 7, - 'iree_community.md': 8, - # 'Design Docs' is 9. - # 'IR Conversion Examples' is 10. - # 'Dialect Definitions' is 11. - - # Within 'Getting Started' use explicit ordering. - # Alphabetical would put 'bazel' before 'cmake' and 'python' between 'linux' - # and 'windows'. - 'getting_started_linux_cmake.md': 1, - 'getting_started_linux_bazel.md': 2, - 'getting_started_linux_vulkan.md': 3, - 'getting_started_windows_cmake.md': 4, - 'getting_started_windows_bazel.md': 5, - 'getting_started_windows_vulkan.md': 6, - 'getting_started_macos_cmake.md': 7, - 'getting_started_macos_bazel.md': 8, - 'getting_started_android_cmake.md': 9, - 'getting_started_python.md': 10, - 'getting_started_tensorflow.md': 11, - 'generic_vulkan_env_setup.md': 12, - 'cmake_options_and_variables.md': 13, - - # Within 'Developing IREE' use explicit ordering. - 'developer_overview.md': 1, - 'contributor_tips.md': 2, - 'testing_guide.md': 3, - 'benchmarking.md': 4, - 'tensorflow_integrations.md': 5, - 'e2e_benchmarking.md': 6, - 'repository_management.md': 7, - - # Within 'TensorFlow Coverage' use explicit ordering. - 'tf_base_coverage.md': 1, - 'tf_keras_coverage.md': 2, - 'language_and_speech_coverage.md': 3, - 'vision_coverage.md': 4, -} - -# A dictionary containing source directory to section tile mappings. -# -# To put a MarkDown file under a certain section, the front matter should -# contain a `parent` field pointing to the section's title. By default we -# use the subdirectory name as the section title. This allows customization. -# Note that the title here must match with index.md file's title under the -# subdirectory. -DIRECTORY_TITLE_DICT = { - 'design_docs': 'Design Docs', - 'developing_iree': 'Developing IREE', - 'Dialects': 'Dialect Definitions', - 'get_started': 'Getting Started', - 'ir_examples': 'IR Conversion Examples', - 'tensorflow_coverage': 'TensorFlow Coverage', -} - -# A dictionary containing the supporting JavaScript files for each doc. -JS_FILES_DICT = { - 'xla_op_coverage.md': ['js/add_classes.js'], - 'tf_base_coverage.md': ['js/add_classes.js'], - 'tf_keras_coverage.md': ['js/add_classes.js'], - 'language_and_speech_coverage.md': ['js/add_classes.js'], - 'vision_coverage.md': ['js/add_classes.js'], -} - - -def process_file(basedir, relpath, filename): - """Patches the given file in-place with metadata for publication.""" - - full_path = os.path.join(basedir, relpath, filename) - base_name = os.path.splitext(filename)[0] - with open(full_path, 'r', encoding="utf8") as f: - content = f.read() - - # Directly return if the file already has front matter. - if content.startswith('---\n'): - return - - front_matter = {} - # Use the default layout for everything. - front_matter['layout'] = 'default' - # Use the base filename as permanent link. - # Replace '_' with '-'. Underscores are not typical in URLs... - front_matter['permalink'] = base_name.replace('_', '-') - - # Organize each doc to a section matching its directory structure. - if relpath and relpath != '.': - hyphen_relpath = relpath.replace('_', '-') - front_matter['permalink'] = f'{hyphen_relpath}/{front_matter["permalink"]}' - - # Find the title and TOC. - lines = content.splitlines() - title_line_index = None - toc_index = None - for (index, line) in enumerate(lines): - if line.startswith('# ') and title_line_index is None: - title_line_index = index - if line == '[TOC]': - toc_index = index - if title_line_index is not None and toc_index is not None: - break - - # Replace '[TOC]' with the proper format that can be rendered. - if toc_index is not None: - lines[toc_index] = '1. TOC\n{:toc}' - - # Set the title in front matter and disable it to show up in TOC. - if title_line_index is not None: - front_matter['title'] = f'"{lines[title_line_index][2:]}"' - lines.insert(title_line_index + 1, '{: .no_toc }') - else: - front_matter['title'] = base_name - - # Override with manually specified metadata if exists. - if filename in DOC_TITLE_DICT: - front_matter['title'] = DOC_TITLE_DICT[filename] - if filename in PERMALINK_DICT: - front_matter['permalink'] = PERMALINK_DICT[filename] - if filename in NAVI_ORDER_DICT: - front_matter['nav_order'] = NAVI_ORDER_DICT[filename] - if relpath in DIRECTORY_TITLE_DICT: - front_matter['parent'] = DIRECTORY_TITLE_DICT[relpath] - if filename in JS_FILES_DICT: - # js_files is a list, so we split each file onto a different line. We put - # an extra \n on the front so the code below can treat it like the rest of - # the values in front_matter. - front_matter['js_files'] = '\n' + '\n'.join( - [f'- {file}' for file in JS_FILES_DICT[filename]]) - - # Compose the content prefix for front matter. - prefix = '\n'.join([f'{k}: {v}' for (k, v) in front_matter.items()]) - prefix = '\n'.join(['---', prefix, '---\n\n']) - - # Compose the new content. - content = '\n'.join(lines) - - # Substitute specific pattern for callouts to make them prettier. - content = content.replace('> Tip:
\n>     ', - '> Tip\n> {: .label .label-green }\n> ') - content = content.replace('> Note:
\n>     ', - '> Note\n> {: .label .label-blue }\n> ') - - # Update in place. - with open(full_path, 'w', encoding="utf8") as f: - f.write(f'{prefix}{content}') - - -def process_directory(basedir): - """Walks the given base directory and processes each MarkDown file.""" - for (dirpath, _, filenames) in os.walk(basedir): - for filename in filenames: - if filename.endswith('.md'): - relpath = os.path.relpath(dirpath, basedir) - process_file(basedir, relpath, filename) - - -if __name__ == '__main__': - args = parse_arguments() - process_directory(args.base_dir) diff --git a/scripts/update_op_coverage.py b/scripts/update_op_coverage.py deleted file mode 100755 index 711fb9039dd4..000000000000 --- a/scripts/update_op_coverage.py +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Updates op coverage table. - -Example usage: ./update_op_coverage.py IREE_BUILD_DIR -""" - -import argparse -import collections -import os -import subprocess - -import utils - -# The symbols to show in the table if the operation is supported or not. -SUCCESS_ELEMENT = '✓' -FAILURE_ELEMENT = '✗' - -E2E_XLA_OPS_PATH = 'iree/test/e2e/xla_ops' - -OP_COVERAGE_DESCRIPTION = """# XLA HLO Op Coverage -There are three backend [targets](https://github.com/google/iree/tree/main/iree/compiler/Dialect/HAL/Target) in IREE: - -- vmla -- dylib-llvm-aot -- vulkan-spirv - -The table shows the supported XLA HLO ops on each backend. It is auto-generated -from IREE's test status. - -""" - - -def parse_arguments(): - """Parses command-line options.""" - parser = argparse.ArgumentParser( - description='Generates Markdown files for op coverage table') - parser.add_argument('build_dir', - metavar='BUILD_PATH', - type=str, - help='Base build directory.') - - parsed_args = parser.parse_args() - if not os.path.isdir(parsed_args.build_dir): - raise parser.error('expected path to a directory') - - return parsed_args - - -def get_backend_op_pair(test): - """Returns the target backend and operation pair of the test.""" - test_suite_backends = { - 'check_vmla_vmla': 'vmla', - 'check_dylib-llvm-aot_dylib': 'dylib-llvm-aot', - 'check_vulkan-spirv_vulkan': 'vulkan-spirv' - } - for (test_suite, backend) in test_suite_backends.items(): - if test_suite in test: - # Format: ...TEST_SUITE_OP.mlir - start_idx = test.index(test_suite) + len(test_suite) + 1 - return backend, test[start_idx:-len('.mlir')] - raise LookupError(f'Can not find a backend to match {test}') - - -def get_tested_ops_for_backends(build_dir): - """Parses current op tests for each backend.""" - - completed_process = subprocess.run( - ['ctest', '-N', '-L', E2E_XLA_OPS_PATH], - cwd=build_dir, - # TODO(#4131) python>=3.7: Use capture_output=True. - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - # TODO(#4131) python>=3.7: Replace 'universal_newlines' with 'text'. - universal_newlines=True) - tests = completed_process.stdout.strip().split('\n') - res = collections.defaultdict(list) - for t in tests: - if not t.endswith('.mlir'): - continue - try: - backend, op = get_backend_op_pair(t) - except LookupError: - # Linalg on tensors are WIP; explicitly ignore them for now. - if "linalg_on_tensors" in t: - continue - raise - res[backend].append(op) - return res - - -def generate_table(build_dir): - """Generates an op coverage Markdown table for each backend.""" - backend_ops = get_tested_ops_for_backends(build_dir) - backends = list(backend_ops.keys()) - - all_ops = [] - for ops in backend_ops.values(): - all_ops.extend(ops) - all_ops = list(set(all_ops)) - all_ops.sort() - - first_row = ['op'] + backends - second_row = [':-:' for _ in first_row] - rows = [first_row, second_row] - for op in all_ops: - row = [op] - for backend in backends: - row.append(SUCCESS_ELEMENT if ( - op in backend_ops[backend]) else FAILURE_ELEMENT) - rows.append(row) - return utils.create_markdown_table(rows) - - -if __name__ == '__main__': - args = parse_arguments() - content = generate_table(args.build_dir) - table_path = os.path.join(args.build_dir, 'doc', 'xla_op_coverage.md') - with open(table_path, 'w', encoding='utf-8') as f: - f.write(OP_COVERAGE_DESCRIPTION) - f.write(content)