Skip to content

Commit 6bfe8c2

Browse files
sadra-barikbinsadra-barikbin
committed
Merge branch 'master' into cocomap
2 parents 07a75cb + f61364f commit 6bfe8c2

File tree

172 files changed

+1197
-1043
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

172 files changed

+1197
-1043
lines changed

‎.circleci/config.yml‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -187,11 +187,6 @@ jobs:
187187
export mnist4_cmd='CUDA_VISIBLE_DEVICES=0 python examples/mnist/mnist_save_resume_engine.py --epochs=2 --resume_from=/tmp/mnist_save_resume/checkpoint_1.pt'
188188
docker exec -it pthd /bin/bash -c "$mnist4_cmd"
189189
190-
- run:
191-
name: Codecov upload
192-
command: |
193-
bash <(curl -s https://codecov.io/bash) -Z -F gpu
194-
195190
one_gpu_windows_tests:
196191
<<: *one_gpu_windows
197192

@@ -258,11 +253,6 @@ jobs:
258253
export test_cmd='bash tests/run_gpu_tests.sh 2'
259254
docker exec -it pthd /bin/bash -c "${test_cmd}"
260255
261-
- run:
262-
name: Codecov upload
263-
command: |
264-
bash <(curl -s https://codecov.io/bash) -Z -F gpu-2
265-
266256
two_gpus_check_dist_cifar10_example:
267257
<<: *two_gpus
268258

@@ -353,11 +343,6 @@ jobs:
353343
export test_cmd='CUDA_VISIBLE_DEVICES="" pytest --cov ignite --cov-append --cov-report term-missing --cov-report xml -vvv tests/ -m distributed -k hvd'
354344
docker exec -it pthd /bin/bash -c "${test_cmd}"
355345
356-
- run:
357-
name: Codecov upload
358-
command: |
359-
bash <(curl -s https://codecov.io/bash) -Z -F gpu-2-hvd
360-
361346
- run:
362347
name: "Check CIFAR10 using horovodrun"
363348
command: |

‎.github/workflows/gpu-tests.yml‎

Lines changed: 130 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -16,95 +16,165 @@ concurrency:
1616
group: gpu-tests-${{ github.ref_name }}-${{ !(github.ref_protected) || github.sha }}
1717
cancel-in-progress: true
1818

19+
# Cherry-picked from https://github.com/pytorch/test-infra/blob/main/.github/workflows/linux_job.yml
20+
1921
jobs:
20-
gpu-tests:
21-
runs-on: [self-hosted, 2-gpus]
22-
timeout-minutes: 45
23-
defaults:
24-
run:
25-
shell: bash
22+
gpu-tests:
2623
strategy:
27-
max-parallel: 1
28-
fail-fast: true
2924
matrix:
3025
pytorch-channel: [pytorch, pytorch-nightly]
26+
fail-fast: false
3127
env:
32-
AGENT_TOOLSDIRECTORY: /tmp/python
28+
DOCKER_IMAGE: "pytorch/conda-builder:cuda11.7"
29+
REPOSITORY: ${{ github.repository }}
30+
PR_NUMBER: ${{ github.event.pull_request.number }}
31+
runs-on: linux.8xlarge.nvidia.gpu
32+
timeout-minutes: 45
3333

3434
steps:
35-
- uses: actions/checkout@v3
36-
37-
- name: Clean python tool path
35+
- name: Clean workspace
3836
run: |
39-
rm -rf ${AGENT_TOOLSDIRECTORY}
37+
echo "::group::Cleanup debug output"
38+
sudo rm -rfv "${GITHUB_WORKSPACE}"
39+
mkdir -p "${GITHUB_WORKSPACE}"
40+
echo "::endgroup::"
41+
42+
- name: Checkout repository (pytorch/test-infra)
43+
uses: actions/checkout@v3
44+
with:
45+
# Support the use case where we need to checkout someone's fork
46+
repository: pytorch/test-infra
47+
path: test-infra
48+
49+
- name: Setup Linux
50+
uses: ./test-infra/.github/actions/setup-linux
4051

41-
- uses: actions/setup-python@v4
52+
- name: Pull docker image
53+
uses: ./test-infra/.github/actions/pull-docker-image
4254
with:
43-
python-version: 3.9
55+
docker-image: ${{ env.DOCKER_IMAGE }}
4456

45-
- name: Install PyTorch
46-
# https://pytorch.org/get-started/locally/
47-
if: ${{ matrix.pytorch-channel == 'pytorch' }}
57+
- name: Checkout repository (${{ github.repository }})
58+
uses: actions/checkout@v3
59+
with:
60+
# Support the use case where we need to checkout someone's fork
61+
repository: ${{ github.repository }}
62+
ref: ${{ github.ref }}
63+
path: ${{ github.repository }}
64+
fetch-depth: 1
65+
66+
- name: Start Pytorch container
67+
working-directory: ${{ github.repository }}
4868
run: |
49-
pip install --upgrade torch torchvision --extra-index-url https://download.pytorch.org/whl/cu117
50-
nvidia-smi
51-
python -c "import torch; print(torch.__version__, ', CUDA is available: ', torch.cuda.is_available()); exit(not torch.cuda.is_available())"
52-
pip list
69+
docker run --name pthd --gpus=all --rm \
70+
--cap-add=SYS_PTRACE \
71+
--detach \
72+
--ipc=host \
73+
--security-opt seccomp=unconfined \
74+
--shm-size=2g \
75+
--tty \
76+
--ulimit stack=10485760:83886080 \
77+
-v $PWD:/work \
78+
-w /work \
79+
${DOCKER_IMAGE}
80+
81+
script=$(cat << EOF
82+
83+
set -x
84+
85+
nvidia-smi
86+
ls -alh
5387
54-
- name: Install PyTorch (nightly)
55-
# https://pytorch.org/get-started/locally/
56-
if: ${{ matrix.pytorch-channel == 'pytorch-nightly' }}
88+
conda --version
89+
python --version
90+
91+
EOF
92+
)
93+
docker exec -t pthd /bin/bash -c "${script}"
94+
95+
- name: Install PyTorch and dependencies
96+
continue-on-error: false
5797
run: |
58-
pip install --upgrade --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu117
98+
99+
script=$(cat << EOF
100+
101+
set -x
102+
103+
# Install PyTorch
104+
if [ "${{ matrix.pytorch-channel }}" == "pytorch" ]; then
105+
pip install --upgrade torch torchvision --extra-index-url https://download.pytorch.org/whl/cu117
106+
else
107+
pip install --upgrade --pre torch torchvision --extra-index-url https://download.pytorch.org/whl/nightly/cu117
108+
fi
109+
59110
nvidia-smi
60111
python -c "import torch; print(torch.__version__, ', CUDA is available: ', torch.cuda.is_available()); exit(not torch.cuda.is_available())"
61112
pip list
62113
63-
- name: Install dependencies
64-
run: |
114+
# Install dependencies
65115
pip install -r requirements-dev.txt
66116
pip install -e .
67117
118+
EOF
119+
)
120+
121+
docker exec -t pthd /bin/bash -c "${script}"
122+
68123
- name: Run 1 Node 2 GPUs Unit Tests
124+
continue-on-error: false
69125
run: |
126+
127+
script=$(cat << EOF
128+
129+
set -x
130+
70131
bash tests/run_gpu_tests.sh 2
71132
133+
EOF
134+
)
135+
136+
docker exec -t pthd /bin/bash -c "${script}"
137+
72138
- name: Upload coverage to Codecov
73139
uses: codecov/codecov-action@v3
74140
with:
75-
file: ./coverage.xml
141+
file: ${{ github.repository }}/coverage.xml
76142
flags: gpu-2
77143
fail_ci_if_error: false
78144

79-
- name: Install additional example dependencies
80-
run: pip install fire
81-
82-
- name: Check training on cifar10, run without backend
83-
run: |
84-
export example_path="examples/contrib/cifar10"
85-
# initial run
86-
export stop_cmd="--stop_iteration=500"
87-
CI=1 python ${example_path}/main.py run --checkpoint_every=200 ${stop_cmd}
88-
# resume
89-
export resume_opt="--resume-from=/tmp/output-cifar10/resnet18_backend-None-1_stop-on-500/training_checkpoint_400.pt"
90-
CI=1 python ${example_path}/main.py run --checkpoint_every=200 --num_epochs=7 ${resume_opt}
91-
92-
- name: Check training on cifar10, run with NCCL backend using torchrun
93-
run: |
94-
export example_path="examples/contrib/cifar10"
95-
# initial run
96-
export stop_cmd="--stop_iteration=500"
97-
CI=1 torchrun --nproc_per_node=2 ${example_path}/main.py run --backend=nccl --checkpoint_every=200 ${stop_cmd}
98-
# resume
99-
export resume_opt="--resume-from=/tmp/output-cifar10/resnet18_backend-nccl-2_stop-on-500/training_checkpoint_400.pt"
100-
CI=1 torchrun --nproc_per_node=2 ${example_path}/main.py run --backend=nccl --checkpoint_every=200 --num_epochs=7 ${resume_opt}
101-
102-
- name: Check training on cifar10, run with NCCL backend using spawn
145+
- name: Run examples in container
146+
continue-on-error: false
103147
run: |
104-
export example_path="examples/contrib/cifar10"
105-
# initial run
106-
export stop_cmd="--stop_iteration=500"
107-
CI=1 python -u ${example_path}/main.py run --backend=nccl --nproc_per_node=2 --checkpoint_every=200 ${stop_cmd}
108-
# resume
109-
export resume_opt="--resume-from=/tmp/output-cifar10/resnet18_backend-nccl-2_stop-on-500/training_checkpoint_400.pt"
110-
CI=1 python -u ${example_path}/main.py run --backend=nccl --nproc_per_node=2 --checkpoint_every=200 --num_epochs=7 ${resume_opt}
148+
SCRIPT=$(cat << EOF
149+
150+
set -x
151+
152+
# Install additional example dependencies
153+
pip install fire
154+
155+
# Check training on cifar10, run without backend
156+
## initial run
157+
CI=1 python examples/contrib/cifar10/main.py run --checkpoint_every=200 --stop_iteration=500
158+
## resume
159+
CI=1 python examples/contrib/cifar10/main.py run --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-None-1_stop-on-500/training_checkpoint_400.pt
160+
161+
# Check training on cifar10, run with NCCL backend using torchrun
162+
## initial run
163+
CI=1 torchrun --nproc_per_node=2 examples/contrib/cifar10/main.py run --backend=nccl --checkpoint_every=200 --stop_iteration=500
164+
## resume
165+
CI=1 torchrun --nproc_per_node=2 examples/contrib/cifar10/main.py run --backend=nccl --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-nccl-2_stop-on-500/training_checkpoint_400.pt
166+
167+
# Check training on cifar10, run with NCCL backend using spawn
168+
## initial run
169+
CI=1 python -u examples/contrib/cifar10/main.py run --backend=nccl --nproc_per_node=2 --checkpoint_every=200 --stop_iteration=500
170+
## resume
171+
CI=1 python -u examples/contrib/cifar10/main.py run --backend=nccl --nproc_per_node=2 --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-nccl-2_stop-on-500/training_checkpoint_400.pt
172+
173+
EOF
174+
)
175+
176+
docker exec -t pthd /bin/bash -c "${script}"
177+
178+
- name: Teardown Linux
179+
if: ${{ always() }}
180+
uses: ./test-infra/.github/actions/teardown-linux

‎.github/workflows/hvd-tests.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ concurrency:
2525
jobs:
2626
horovod-tests:
2727
runs-on: ubuntu-latest
28+
timeout-minutes: 60
2829
strategy:
2930
matrix:
3031
python-version: [3.8]

‎.github/workflows/pytorch-version-tests.yml‎

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -16,40 +16,41 @@ jobs:
1616
matrix:
1717
python-version: [3.8, 3.9, "3.10"]
1818
pytorch-version:
19-
[1.12.1, 1.11.0, 1.10.0, 1.9.1, 1.8.1, 1.7.1, 1.6.0, 1.5.1, 1.4.0, 1.3.1]
19+
[1.13.1, 1.12.1, 1.11.0, 1.10.0, 1.9.1, 1.8.1, 1.7.1, 1.6.0, 1.5.1, 1.4.0]
2020
exclude:
21-
- pytorch-version: 1.3.1
22-
python-version: 3.8
23-
- pytorch-version: 1.3.1
24-
python-version: 3.9
2521
- pytorch-version: 1.4.0
2622
python-version: 3.9
23+
- pytorch-version: 1.4.0
24+
python-version: 3.10
25+
2726
- pytorch-version: 1.5.1
2827
python-version: 3.9
28+
- pytorch-version: 1.5.1
29+
python-version: 3.10
30+
2931
- pytorch-version: 1.6.0
3032
python-version: 3.9
33+
- pytorch-version: 1.6.0
34+
python-version: 3.10
35+
3136
# disabling python 3.9 support with PyTorch 1.7.1 and 1.8.1, to stop repeated pytorch-version test fail.
3237
# https://github.com/pytorch/ignite/issues/2383
3338
- pytorch-version: 1.7.1
3439
python-version: 3.9
35-
- pytorch-version: 1.8.1
36-
python-version: 3.9
37-
- pytorch-version: 1.3.1
38-
python-version: 3.10
39-
- pytorch-version: 1.4.0
40-
python-version: 3.10
41-
- pytorch-version: 1.5.1
42-
python-version: 3.10
43-
- pytorch-version: 1.6.0
44-
python-version: 3.10
4540
- pytorch-version: 1.7.1
4641
python-version: 3.10
42+
43+
- pytorch-version: 1.8.1
44+
python-version: 3.9
4745
- pytorch-version: 1.8.1
4846
python-version: 3.10
47+
4948
- pytorch-version: 1.9.1
5049
python-version: 3.10
50+
5151
- pytorch-version: 1.10.0
5252
python-version: 3.10
53+
5354
- pytorch-version: 1.11.0
5455
python-version: 3.10
5556

@@ -86,17 +87,22 @@ jobs:
8687

8788
- name: Install dependencies
8889
shell: bash -l {0}
90+
if: ${{ matrix.pytorch-version != '1.4.0' }}
8991
run: |
9092
conda install pytorch=${{ matrix.pytorch-version }} torchvision cpuonly python=${{ matrix.python-version }} -c pytorch
9193
pip install -r requirements-dev.txt
9294
python setup.py install
9395
94-
- name: Install appropriate Pillow for PyTorch 1.3.1
96+
# There is no more torchvision 0.5.0 binaries in anaconda pytorch channel:
97+
# https://anaconda.org/pytorch/torchvision/files
98+
- name: Install appropriate dependencies for PyTorch 1.4.0
9599
shell: bash -l {0}
96-
if: ${{ matrix.pytorch-version == '1.3.1' }}
97-
run: |
98-
pip install --upgrade 'Pillow<7'
99-
python -c "import torchvision"
100+
if: ${{ matrix.pytorch-version == '1.4.0' }}
101+
run: |
102+
conda install pytorch=${{ matrix.pytorch-version }} cpuonly python=${{ matrix.python-version }} -c pytorch
103+
pip install torchvision==0.5.0
104+
pip install -r requirements-dev.txt
105+
python setup.py install
100106
101107
- name: Download MNIST
102108
uses: pytorch-ignite/download-mnist-github-action@master

‎.github/workflows/trigger_circle_ci.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ def assert_pipeline_created(pipeline_id, headers):
4242

4343

4444
def get_workflow_id(pipeline_id, headers):
45-
4645
while True:
4746
result = requests.get(f"https://circleci.com/api/v2/pipeline/{pipeline_id}/workflow", headers=headers)
4847
assert_result(result, 200)
@@ -59,7 +58,6 @@ def get_workflow_id(pipeline_id, headers):
5958

6059

6160
def assert_workflows_successful(pipeline_id, headers):
62-
6361
workflow_id = get_workflow_id(pipeline_id, headers)
6462

6563
base_url = "https://app.circleci.com/pipelines/github/pytorch/ignite"
@@ -84,7 +82,6 @@ def assert_workflows_successful(pipeline_id, headers):
8482

8583

8684
if __name__ == "__main__":
87-
8885
print("Trigger new pipeline on Circle-CI")
8986

9087
if "CIRCLE_TOKEN" not in os.environ:

0 commit comments

Comments
 (0)