Skip to content

Commit c377f98

Browse files
liligwuliligwu
authored
Merge pull request ROCm#30 from ROCmSoftwarePlatform/update-from-public
Update from public repo 2022-08-02
2 parents c6f77ae + a98d7f4 commit c377f98

File tree

151 files changed

+11675
-3154
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

151 files changed

+11675
-3154
lines changed

‎.github/workflows/fbgemm_docs.yml

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# This workflow builds the fbgemm_gpu docs and deploys them to gh-pages.
2+
name: Generate documentation
3+
on:
4+
push:
5+
branches:
6+
- main
7+
jobs:
8+
build_docs_job:
9+
runs-on: linux.2xlarge
10+
steps:
11+
# Checkout the repository to the GitHub Actions runner
12+
- name: Checkout
13+
uses: actions/checkout@v2
14+
with:
15+
submodules: true
16+
# Update references
17+
# TODO: update the git submodule sync after we fixed the auto-sync part
18+
- name: Git Sumbodule Update
19+
run: |
20+
git submodule init
21+
git submodule update --remote --recursive
22+
git log
23+
- name: Update pip
24+
run: |
25+
sudo yum update -y
26+
sudo yum -y install git python3-pip
27+
sudo pip3 install --upgrade pip
28+
- name: Setup conda
29+
run: |
30+
wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh
31+
bash ~/miniconda.sh -b -p $HOME/miniconda
32+
- name: setup Path
33+
run: |
34+
echo "/home/ec2-user/miniconda/bin" >> $GITHUB_PATH
35+
echo "CONDA=/home/ec2-user/miniconda" >> $GITHUB_PATH
36+
- name: create conda env
37+
run: |
38+
conda create --name build_binary python=3.9
39+
conda info
40+
- name: check python version
41+
run: |
42+
conda run -n build_binary python --version
43+
- name: Install gcc
44+
shell: bash
45+
run: |
46+
sudo yum group install -y "Development Tools"
47+
- name: Setup Path
48+
run: |
49+
echo /usr/local/bin >> $GITHUB_PATH
50+
- name: Install PyTorch
51+
shell: bash
52+
run: |
53+
conda run -n build_binary python -m pip install --pre torch -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html
54+
- name: Test PyTorch Installation
55+
run: |
56+
conda run -n build_binary python -c "import torch.distributed"
57+
echo "torch.distributed succeeded"
58+
- name: Install fbgemm_gpu nightly
59+
run: |
60+
conda run -n build_binary python -m pip install fbgemm-gpu-nightly-cpu
61+
- name: Test fbgemm_gpu installation
62+
shell: bash
63+
run: |
64+
conda run -n build_binary \
65+
python -c "import fbgemm_gpu"
66+
- name: Install Doxygen
67+
run: |
68+
conda install -n build_binary -c conda-forge doxygen
69+
which doxygen
70+
- name: Build the docset
71+
run: |
72+
conda run -n build_binary python -m pip install -r fbgemm_gpu/docs/requirements.txt
73+
cd ./fbgemm_gpu/docs
74+
conda run -n build_binary doxygen Doxyfile.in
75+
conda run -n build_binary make html
76+
cd ..
77+
- name: Get output time
78+
run: echo "The time was ${{ steps.build.outputs.time }}"
79+
- name: Deploy
80+
uses: JamesIves/github-pages-deploy-action@releases/v3
81+
with:
82+
ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
83+
BRANCH: gh-pages # The branch the action should deploy to.
84+
FOLDER: fbgemm_gpu/docs/build/html # The folder the action should deploy.

‎.github/workflows/fbgemm_nightly_build.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ jobs:
117117
--python-tag=${{ matrix.python-tag }} \
118118
-DTORCH_CUDA_ARCH_LIST="'7.0;8.0'" \
119119
--plat-name=manylinux1_x86_64
120+
ls -lt dist/*.whl
120121
- name: Upload wheel as GHA artifact
121122
uses: actions/upload-artifact@v2
122123
with:
@@ -271,4 +272,5 @@ jobs:
271272
--username __token__ \
272273
--password "$PYPI_TOKEN" \
273274
--skip-existing \
275+
--verbose \
274276
fbgemm_gpu_nightly-*.whl

‎.github/workflows/fbgemm_nightly_build_cpu.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ jobs:
105105
--python-tag=${{ matrix.python-tag }} \
106106
--cpu_only \
107107
--plat-name=manylinux1_x86_64
108+
ls -lt dist/*.whl
108109
- name: Upload wheel as GHA artifact
109110
uses: actions/upload-artifact@v2
110111
with:
@@ -156,4 +157,5 @@ jobs:
156157
--username __token__ \
157158
--password "$PYPI_TOKEN" \
158159
--skip-existing \
160+
--verbose \
159161
fbgemm_gpu/dist/fbgemm_gpu_nightly_cpu-*.whl

‎.github/workflows/fbgemm_release_build.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ jobs:
119119
--python-tag=${{ matrix.python-tag }} \
120120
-DTORCH_CUDA_ARCH_LIST="'7.0;8.0'" \
121121
--plat-name=manylinux1_x86_64
122+
ls -lt dist/*.whl
122123
- name: Upload wheel as GHA artifact
123124
uses: actions/upload-artifact@v2
124125
with:
@@ -273,4 +274,5 @@ jobs:
273274
--username __token__ \
274275
--password "$PYPI_TOKEN" \
275276
--skip-existing \
277+
--verbose \
276278
fbgemm_gpu-*.whl

‎.github/workflows/fbgemm_release_build_cpu.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ jobs:
107107
--python-tag=${{ matrix.python-tag }} \
108108
--cpu_only \
109109
--plat-name=manylinux1_x86_64
110+
ls -lt dist/*.whl
110111
- name: Upload wheel as GHA artifact
111112
uses: actions/upload-artifact@v2
112113
with:
@@ -158,4 +159,5 @@ jobs:
158159
--username __token__ \
159160
--password "$PYPI_TOKEN" \
160161
--skip-existing \
162+
--verbose \
161163
fbgemm_gpu/dist/fbgemm_gpu_cpu-*.whl

‎.github/workflows/fbgemmci.yml

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -248,6 +248,113 @@ jobs:
248248
python -c "import fbgemm_gpu"
249249
python -c "import fbgemm_gpu.split_embedding_codegen_lookup_invokers"
250250
251+
build_amd_gpu:
252+
runs-on: ${{ matrix.os }}
253+
strategy:
254+
matrix:
255+
os: [ubuntu-latest]
256+
257+
steps:
258+
- name: Free space
259+
run: sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android
260+
261+
- uses: actions/checkout@v2
262+
263+
- name: Install ROCm 5.1.1
264+
shell: bash
265+
run: |
266+
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10
267+
wget https://repo.radeon.com/amdgpu-install/22.10.1/ubuntu/focal/amdgpu-install_22.10.1.50101-1_all.deb
268+
export DEBIAN_FRONTEND=noninteractive
269+
sudo apt install -y ./amdgpu-install_22.10.1.50101-1_all.deb
270+
amdgpu-install -y --usecase=hiplibsdk,rocm --no-dkms
271+
sudo rm amdgpu-install_22.10.1.50101-1_all.deb
272+
273+
- name: Install dependencies
274+
shell: bash
275+
run: |
276+
sudo apt-get update
277+
sudo apt-get -y install git pip python3-dev mesa-common-dev clang comgr libopenblas-dev jp intel-mkl-full locales libnuma-dev
278+
sudo apt-get install -y hipify-clang || true
279+
sudo pip install cmake scikit-build ninja jinja2 numpy hypothesis --no-input
280+
sudo apt-get clean
281+
# Install pytorch 1.11 as required by fbgemm_gpu
282+
sudo pip install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/rocm5.1.1/
283+
284+
- name: Checkout submodules
285+
shell: bash
286+
run: |
287+
cd fbgemm_gpu
288+
git submodule sync
289+
git submodule update --init --recursive
290+
291+
- name: Build fbgemm_gpu
292+
shell: bash
293+
run: |
294+
sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 10
295+
cd fbgemm_gpu
296+
# build for MI250 only to save time.
297+
sudo PYTORCH_ROCM_ARCH=gfx90a python3 setup.py build develop
298+
299+
- name: Test fbgemm_gpu installation
300+
shell: bash
301+
run: |
302+
cd fbgemm_gpu
303+
cd test
304+
python3 input_combine_test.py
305+
python3 quantize_ops_test.py
306+
python3 sparse_ops_test.py
307+
python3 -c "import fbgemm_gpu"
308+
python3 -c "import fbgemm_gpu.split_embedding_codegen_lookup_invokers"
309+
310+
test_amd_gpu:
311+
runs-on: rocm
312+
strategy:
313+
matrix:
314+
os: [ubuntu-latest]
315+
316+
steps:
317+
- name: pre-checkout
318+
shell: bash
319+
run: |
320+
if [ -d ${{ github.workspace }} ]
321+
then
322+
sudo chown -R $USER:$USER ${{ github.workspace }}
323+
fi
324+
sudo add-apt-repository ppa:git-core/ppa
325+
sudo apt update
326+
sudo apt -y install --only-upgrade git
327+
328+
- uses: actions/checkout@v2
329+
with:
330+
ref: ${{ github.ref }}
331+
submodules: 'true'
332+
333+
- name: build fbgemm_gpu and test
334+
shell: bash
335+
run: |
336+
set -eux
337+
env
338+
ls -l
339+
DOCKER_IMAGE=rocm/pytorch:rocm5.1.1_ubuntu20.04_py3.7_pytorch_staging_base
340+
docker pull $DOCKER_IMAGE
341+
JENKINS_REPO_DIR=fbgemm-private-jenkins
342+
JENKINS_REPO_DIR_BAREMETAL=$PWD
343+
JENKINS_REPO_DIR_DOCKER=/workspace/$JENKINS_REPO_DIR
344+
DOCKER_OPTIONS="\
345+
--user 0 \
346+
--network=host \
347+
--ipc=host \
348+
--shm-size 16G \
349+
--group-add video \
350+
--cap-add=SYS_PTRACE \
351+
--security-opt seccomp=unconfined \
352+
--device=/dev/kfd \
353+
--device=/dev/dri \
354+
-v $JENKINS_REPO_DIR_BAREMETAL:$JENKINS_REPO_DIR_DOCKER
355+
"
356+
docker run $DOCKER_OPTIONS $DOCKER_IMAGE $JENKINS_REPO_DIR_DOCKER/.jenkins/rocm/build_and_test.sh $JENKINS_REPO_DIR_DOCKER
357+
251358
build_cpu_only:
252359
runs-on: ${{ matrix.os }}
253360
strategy:

‎.jenkins/rocm/build_and_test.sh

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
#!/bin/bash
2+
3+
# exit immediately on failure, or if an undefined variable is used
4+
set -eux
5+
6+
FBGEMM_REPO_DIR=${1:-/workspace/FBGEMM}
7+
8+
git config --global --add safe.directory "$FBGEMM_REPO_DIR"
9+
git config --global --add safe.directory "$FBGEMM_REPO_DIR/third_party/asmjit"
10+
git config --global --add safe.directory "$FBGEMM_REPO_DIR/third_party/cpuinfo"
11+
git config --global --add safe.directory "$FBGEMM_REPO_DIR/third_party/googletest"
12+
git config --global --add safe.directory "$FBGEMM_REPO_DIR/third_party/hipify_torch"
13+
14+
# Install dependencies
15+
apt-get update --allow-insecure-repositories && \
16+
apt-get install -y --allow-unauthenticated \
17+
git \
18+
jq \
19+
sshfs \
20+
sshpass \
21+
unzip
22+
23+
apt-get install -y locales
24+
locale-gen en_US.UTF-8
25+
26+
pip3 install click
27+
pip3 install jinja2
28+
pip3 install ninja
29+
pip3 install scikit-build
30+
pip3 install --upgrade hypothesis
31+
pip3 install --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/rocm5.1.1/
32+
33+
pip3 list
34+
35+
# Build fbgemm_gpu
36+
cd "$FBGEMM_REPO_DIR/fbgemm_gpu"
37+
MAX_JOBS="$(nproc)"
38+
export MAX_JOBS
39+
export PYTORCH_ROCM_ARCH="gfx908"
40+
python setup.py build develop
41+
42+
export FBGEMM_TEST_WITH_ROCM=1
43+
44+
# Test fbgemm_gpu
45+
cd test
46+
47+
python batched_unary_embeddings_test.py --verbose
48+
python input_combine_test.py --verbose
49+
python jagged_tensor_ops_test.py --verbose
50+
python layout_transform_ops_test.py --verbose
51+
python merge_pooled_embeddings_test.py --verbose
52+
python metric_ops_test.py --verbose
53+
python permute_pooled_embedding_modules_test.py --verbose
54+
python quantize_ops_test.py --verbose
55+
python sparse_ops_test.py --verbose
56+
python split_embedding_inference_converter_test.py --verbose
57+
python split_table_batched_embeddings_test.py --verbose
58+
python uvm_test.py --verbose

‎CMakeLists.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,10 @@ if(NOT TARGET asmjit)
182182

183183
add_subdirectory("${ASMJIT_SRC_DIR}" "${FBGEMM_BINARY_DIR}/asmjit")
184184
set_property(TARGET asmjit PROPERTY POSITION_INDEPENDENT_CODE ON)
185+
# add a flag required for mac build
186+
if(NOT MSVC)
187+
target_compile_options(asmjit PRIVATE "-Wno-sign-conversion")
188+
endif()
185189
endif()
186190

187191
if(NOT TARGET cpuinfo)
@@ -293,6 +297,10 @@ endif()
293297

294298
if(FBGEMM_BUILD_BENCHMARKS)
295299
add_subdirectory(bench)
300+
# add a flag to enable Clang 14
301+
set_source_files_properties(
302+
bench/GEMMsBenchmark.cc
303+
PROPERTIES COMPILE_FLAGS "-Wno-unused-variable")
296304
endif()
297305

298306
if(FBGEMM_BUILD_DOCS)

‎README.md

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -72,9 +72,8 @@ cd FBGEMM
7272
# if you are updating an existing checkout
7373
git submodule sync
7474
git submodule update --init --recursive
75-
mkdir build && cd build
76-
cmake ..
77-
make
75+
cmake -B build
76+
make -C build
7877
```
7978

8079
To run the tests after building FBGEMM (if tests are built), use the following

0 commit comments

Comments
 (0)