Merge branch 'dev' into dev

ssheorey · web-flow · commit 16f89cca8057 · 2023-12-23T03:08:30.000+05:30
diff --git a/.github/ISSUE_TEMPLATE/0-build-issue-report.yml b/.github/ISSUE_TEMPLATE/0-build-issue-report.yml
@@ -13,7 +13,7 @@ body:
         required: true
       - label: "I have tested with the [latest development wheel](http://www.open3d.org/docs/latest/getting_started.html#development-version-pip)."
         required: true
-      - label: "I have checked the [release documentation](http://www.open3d.org/docs/release/) and the [latest documentation](http://www.open3d.org/docs/latest/) (for `master` branch)."
+      - label: "I have checked the [release documentation](http://www.open3d.org/docs/release/) and the [latest documentation](http://www.open3d.org/docs/latest/) (for `main` branch)."
         required: true
 
 - type: textarea
diff --git a/.github/ISSUE_TEMPLATE/1-bug-report.yml b/.github/ISSUE_TEMPLATE/1-bug-report.yml
@@ -13,7 +13,7 @@ body:
         required: true
       - label: "I have tested with the [latest development wheel](http://www.open3d.org/docs/latest/getting_started.html#development-version-pip)."
         required: true
-      - label: "I have checked the [release documentation](http://www.open3d.org/docs/release/) and the [latest documentation](http://www.open3d.org/docs/latest/) (for `master` branch)."
+      - label: "I have checked the [release documentation](http://www.open3d.org/docs/release/) and the [latest documentation](http://www.open3d.org/docs/latest/) (for `main` branch)."
         required: true
 
 - type: textarea
diff --git a/.github/ISSUE_TEMPLATE/2-questions.yml b/.github/ISSUE_TEMPLATE/2-questions.yml
@@ -13,7 +13,7 @@ body:
         required: true
       - label: "I have tested with the [latest development wheel](http://www.open3d.org/docs/latest/getting_started.html#development-version-pip)."
         required: true
-      - label: "I have checked the [release documentation](http://www.open3d.org/docs/release/) and the [latest documentation](http://www.open3d.org/docs/latest/) (for `master` branch)."
+      - label: "I have checked the [release documentation](http://www.open3d.org/docs/release/) and the [latest documentation](http://www.open3d.org/docs/latest/) (for `main` branch)."
         required: true
 
 - type: textarea
diff --git a/.github/ISSUE_TEMPLATE/3-feature-request.yml b/.github/ISSUE_TEMPLATE/3-feature-request.yml
@@ -13,7 +13,7 @@ body:
         required: true
       - label: "I have tested with the [latest development wheel](http://www.open3d.org/docs/latest/getting_started.html#development-version-pip)."
         required: true
-      - label: "I have checked the [release documentation](http://www.open3d.org/docs/release/) and the [latest documentation](http://www.open3d.org/docs/latest/) (for `master` branch)."
+      - label: "I have checked the [release documentation](http://www.open3d.org/docs/release/) and the [latest documentation](http://www.open3d.org/docs/latest/) (for `main` branch)."
         required: true
 
 - type: textarea
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -3,7 +3,7 @@ name: Style check
 on:
   workflow_dispatch:
   push:
-    branches: [master, dev]
+    branches: [main, dev]
   pull_request:
     types: [opened, reopened, synchronize]
 
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -3,7 +3,7 @@ name: Ubuntu CI
 on:
   workflow_dispatch:
   push:
-    branches: [master, dev]
+    branches: [main, dev]
   pull_request:
     types: [opened, reopened, synchronize]
 
@@ -16,7 +16,7 @@ jobs:
       NPROC: 2
     steps:
       - name: Checkout source code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup cache
         uses: actions/cache@v3
         with:
@@ -33,7 +33,7 @@ jobs:
       - name: Set up Python version
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       # Pre-installed  packages: https://github.com/actions/runner-images/tree/main/images
       - name: Install ccache
         run: |
diff --git a/README.md b/README.md
diff --git a/ci/run_ci.sh b/ci/run_ci.sh
@@ -1,54 +1,42 @@
 #!/usr/bin/env bash
-#
-# The following environment variables are required:
-# - NPROC
-#
-TENSORFLOW_VER="2.8.2"
-TORCH_GLNX_VER="1.13.1+cpu"
-# OPENVINO_DEV_VER="2021.4.2"  # Numpy version conflict with TF 2.8.2
-PIP_VER="21.1.1"
-WHEEL_VER="0.38.4"
-STOOLS_VER="67.3.2"
-YAPF_VER="0.30.0"
-PYTEST_VER="7.1.2"
-PYTEST_RANDOMLY_VER="3.8.0"
 
 set -euo pipefail
 
+NPROC=${NPROC:?'env var must be set to number of available CPUs.'}
+PIP_VER="23.2.1"
+
 echo 1. Prepare the Open3D-ML repo and install dependencies
 echo
-export PATH_TO_OPEN3D_ML=$(pwd)
-# the build system of the main repo expects a master branch. make sure master exists
-git checkout -b master || true
-python -m pip install -U pip==$PIP_VER \
-    wheel=="$WHEEL_VER" \
-    setuptools=="$STOOLS_VER" \
-    yapf=="$YAPF_VER" \
-    pytest=="$PYTEST_VER" \
-    pytest-randomly=="$PYTEST_RANDOMLY_VER"
-
-python -m pip install -r requirements.txt
-echo $PATH_TO_OPEN3D_ML
+export PATH_TO_OPEN3D_ML="$PWD"
+echo "$PATH_TO_OPEN3D_ML"
+# the build system of the main repo expects a main branch. make sure main exists
+git checkout -b main || true
+python -m pip install -U pip==$PIP_VER
+python -m pip install -r requirements.txt \
+    -r requirements-torch.txt
+# -r requirements-tensorflow.txt # TF disabled on Linux (Open3D PR#6288)
+# -r requirements-openvino.txt # Numpy version conflict with TF 2.8.2
 cd ..
 python -m pip install -U Cython
 
 echo 2. clone Open3D and install dependencies
 echo
-git clone --recursive --branch master https://github.com/isl-org/Open3D.git
+git clone --branch main https://github.com/isl-org/Open3D.git
 
 ./Open3D/util/install_deps_ubuntu.sh assume-yes
-python -m pip install -U tensorflow-cpu==$TENSORFLOW_VER \
-    torch==${TORCH_GLNX_VER} --extra-index-url https://download.pytorch.org/whl/cpu/
-# openvino-dev=="$OPENVINO_DEV_VER"
+python -m pip install -r Open3D/python/requirements.txt \
+    -r Open3D/python/requirements_style.txt \
+    -r Open3D/python/requirements_test.txt
 
 echo 3. Configure for bundling the Open3D-ML part
 echo
 mkdir Open3D/build
 pushd Open3D/build
+# TF disabled on Linux (Open3D PR#6288)
 cmake -DBUNDLE_OPEN3D_ML=ON \
-    -DOPEN3D_ML_ROOT=$PATH_TO_OPEN3D_ML \
+    -DOPEN3D_ML_ROOT="${PATH_TO_OPEN3D_ML}" \
     -DGLIBCXX_USE_CXX11_ABI=OFF \
-    -DBUILD_TENSORFLOW_OPS=ON \
+    -DBUILD_TENSORFLOW_OPS=OFF \
     -DBUILD_PYTORCH_OPS=ON \
     -DBUILD_GUI=ON \
     -DBUILD_UNIT_TESTS=OFF \
@@ -66,12 +54,13 @@ echo
 popd
 mkdir test_workdir
 pushd test_workdir
-mv $PATH_TO_OPEN3D_ML/tests .
+mv "$PATH_TO_OPEN3D_ML/tests" .
 echo Add --randomly-seed=SEED to the test command to reproduce test order.
 python -m pytest tests
 
-echo ... now do the same but in dev mode by setting OPEN3D_ML_ROOT
-export OPEN3D_ML_ROOT=$PATH_TO_OPEN3D_ML
+echo "... now do the same but in dev mode by setting OPEN3D_ML_ROOT"
+echo
+export OPEN3D_ML_ROOT="$PATH_TO_OPEN3D_ML"
 echo Add --randomly-seed=SEED to the test command to reproduce test order.
 python -m pytest tests
 unset OPEN3D_ML_ROOT
diff --git a/ml3d/tf/pipelines/object_detection.py b/ml3d/tf/pipelines/object_detection.py
@@ -62,7 +62,7 @@ def run_test(self):
         dataset = self.dataset
         cfg = self.cfg
 
-        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
 
         log_file_path = join(cfg.logs_dir, 'log_test_' + timestamp + '.txt')
         log.info("Logging in file : {}".format(log_file_path))
@@ -111,7 +111,7 @@ def run_valid(self, epoch=0):
         dataset = self.dataset
         cfg = self.cfg
 
-        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
 
         log_file_path = join(cfg.logs_dir, 'log_valid_' + timestamp + '.txt')
         log.info("Logging in file : {}".format(log_file_path))
@@ -212,7 +212,7 @@ def run_train(self):
 
         cfg = self.cfg
 
-        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
         log_file_path = join(cfg.logs_dir, 'log_train_' + timestamp + '.txt')
         log.info("Logging in file : {}".format(log_file_path))
         log.addHandler(logging.FileHandler(log_file_path))
diff --git a/ml3d/tf/pipelines/semantic_segmentation.py b/ml3d/tf/pipelines/semantic_segmentation.py
@@ -147,7 +147,7 @@ def run_test(self):
         cfg = self.cfg
 
         self.load_ckpt(model.cfg.ckpt_path)
-        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
 
         log_file_path = join(cfg.logs_dir, 'log_test_' + timestamp + '.txt')
         log.info("Logging in file : {}".format(log_file_path))
@@ -192,7 +192,7 @@ def run_train(self):
         cfg = self.cfg
 
         log.info(model)
-        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
         log_file_path = join(cfg.logs_dir, 'log_train_' + timestamp + '.txt')
         log.info("Logging in file : {}".format(log_file_path))
         log.addHandler(logging.FileHandler(log_file_path))
diff --git a/ml3d/torch/pipelines/object_detection.py b/ml3d/torch/pipelines/object_detection.py
@@ -85,7 +85,7 @@ def run_test(self):
 
         model.eval()
 
-        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
 
         log.info("DEVICE : {}".format(device))
         log_file_path = join(cfg.logs_dir, 'log_test_' + timestamp + '.txt')
@@ -145,7 +145,7 @@ def run_valid(self, epoch=0):
 
         model.eval()
 
-        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
 
         log.info("DEVICE : {}".format(device))
         log_file_path = join(cfg.logs_dir, 'log_valid_' + timestamp + '.txt')
@@ -282,7 +282,7 @@ def run_train(self):
 
         if rank == 0:
             log.info("DEVICE : {}".format(device))
-            timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+            timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
 
             log_file_path = join(cfg.logs_dir,
                                  'log_train_' + timestamp + '.txt')
diff --git a/ml3d/torch/pipelines/semantic_segmentation.py b/ml3d/torch/pipelines/semantic_segmentation.py
@@ -190,7 +190,7 @@ def run_test(self):
         model.eval()
         self.metric_test = SemSegMetric()
 
-        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
 
         log.info("DEVICE : {}".format(device))
         log_file_path = join(cfg.logs_dir, 'log_test_' + timestamp + '.txt')
@@ -320,7 +320,7 @@ def run_train(self):
         model.to(device)
 
         log.info("DEVICE : {}".format(device))
-        timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
+        timestamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
 
         log_file_path = join(cfg.logs_dir, 'log_train_' + timestamp + '.txt')
         log.info("Logging in file : {}".format(log_file_path))
diff --git a/model_zoo.md b/model_zoo.md
@@ -34,5 +34,5 @@ The following is a list of datasets for which we provide dataset reader classes.
 * S3DIS ([project-page](http://3dsemantics.stanford.edu/))
 * Paris-Lille 3D ([project-page](https://npm3d.fr/paris-lille-3d))
 
-For downloading these datasets visit the respective webpages and have a look at the scripts in [`scripts/download_datasets`](https://github.com/isl-org/Open3D-ML/tree/master/scripts/download_datasets).
+For downloading these datasets visit the respective webpages and have a look at the scripts in [`scripts/download_datasets`](https://github.com/isl-org/Open3D-ML/tree/main/scripts/download_datasets).
 
diff --git a/requirements-tensorflow.txt b/requirements-tensorflow.txt
@@ -1,2 +1 @@
-tensorflow~=2.8.4 ; sys_platform != 'darwin' or platform_machine != 'arm64'
-tensorflow-macos==2.8.0 ; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow~=2.13.0
diff --git a/requirements-torch-cuda.txt b/requirements-torch-cuda.txt
@@ -1,5 +1,5 @@
 -f https://download.pytorch.org/whl/torch/
-torch==1.13.1+cu116
+torch==2.0.1+cu117
 -f https://download.pytorch.org/whl/torchvision/
-torchvision==0.14.1+cu116
+torchvision==0.15.2+cu117
 tensorboard
diff --git a/requirements-torch.txt b/requirements-torch.txt
@@ -1,6 +1,6 @@
 --extra-index-url https://download.pytorch.org/whl/cpu/
-torch==1.13.1+cpu ; sys_platform != 'darwin'
-torchvision==0.14.1+cpu ; sys_platform != 'darwin'
-torch==1.13.1 ; sys_platform == 'darwin'
-torchvision==0.14.1 ; sys_platform == 'darwin'
+torch==2.0.1+cpu ; sys_platform != 'darwin'
+torchvision==0.15.2+cpu ; sys_platform != 'darwin'
+torch==2.0.1 ; sys_platform == 'darwin'
+torchvision==0.15.2 ; sys_platform == 'darwin'
 tensorboard
diff --git a/requirements.txt b/requirements.txt
@@ -1,7 +1,7 @@
 addict
 pillow>=9.3.0
 matplotlib>=3
-numpy>1.15
+numpy>1.18
 pandas>=1.0
 pyyaml>=5.4.1
 scikit-learn>=0.21
diff --git a/scripts/run_pipeline.py b/scripts/run_pipeline.py
@@ -199,8 +199,8 @@ def main():
 
 
 def setup(rank, world_size, args):
-    os.environ['MASTER_ADDR'] = args.host
-    os.environ['MASTER_PORT'] = args.port
+    os.environ['PRIMARY_ADDR'] = args.host
+    os.environ['PRIMARY_PORT'] = args.port
 
     # initialize the process group
     dist.init_process_group(args.backend, rank=rank, world_size=world_size)
diff --git a/scripts/train_scripts/pointpillars_waymo.sh b/scripts/train_scripts/pointpillars_waymo.sh
@@ -15,9 +15,9 @@ if [ "$#" -ne 2 ]; then
     exit 1
 fi
 
-# Use launch node as master, if not set
-export MASTER_ADDR=${MASTER_ADDR:-$SLURMD_NODENAME}
-export MASTER_PORT=${MASTER_PORT:-29500}
+# Use launch node as main, if not set
+export PRIMARY_ADDR=${PRIMARY_ADDR:-$SLURMD_NODENAME}
+export PRIMARY_PORT=${PRIMARY_PORT:-29500}
 # Use all available GPUs, if not set. Must be the same for ALL nodes.
 export DEVICE_IDS=${DEVICE_IDS:-$(nvidia-smi --list-gpus | cut -f2 -d' ' | tr ':\n' ' ')}
 export NODE_RANK=${NODE_RANK:-SLURM_NODEID} # Pass name of env var
@@ -32,7 +32,7 @@ srun -l python scripts/run_pipeline.py "$1" -c ml3d/configs/pointpillars_waymo.y
     --backend nccl \
     --nodes $SLURM_JOB_NUM_NODES \
     --node_rank "$NODE_RANK" \
-    --host "$MASTER_ADDR" --port "$MASTER_PORT"
+    --host "$PRIMARY_ADDR" --port "$PRIMARY_PORT"
 
 echo Completed at: $(date)
 popd
diff --git a/tests/test_integration.py b/tests/test_integration.py
@@ -1,5 +1,6 @@
 import pytest
 import os
+import open3d as o3d
 
 if 'PATH_TO_OPEN3D_ML' in os.environ.keys():
     base = os.environ['PATH_TO_OPEN3D_ML']
@@ -8,6 +9,7 @@
     # base = '../Open3D-ML'
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_PYTORCH_OPS']")
 def test_integration_torch():
     import torch
     import open3d.ml.torch as ml3d
@@ -25,6 +27,7 @@ def test_integration_torch():
     print(model)
 
 
+@pytest.mark.skipif("not o3d._build_config['BUILD_TENSORFLOW_OPS']")
 def test_integration_tf():
     import tensorflow as tf
     import open3d.ml.tf as ml3d
@@ -45,15 +48,10 @@ def test_integration_tf():
 def test_integration_openvino():
     try:
         from openvino.inference_engine import IECore
-        openvino_available = True
-    except:
-        openvino_available = False
-
-    if not openvino_available:
+    except ImportError:
         return
 
-    from open3d.ml.torch.models import OpenVINOModel
-    from open3d.ml.tf.models import OpenVINOModel
-
-
-test_integration_torch()
+    if o3d._build_config['BUILD_TORCH_OPS']:
+        from open3d.ml.torch.models import OpenVINOModel
+    if o3d._build_config['BUILD_TENSORFLOW_OPS']:
+        from open3d.ml.tf.models import OpenVINOModel
diff --git a/tests/test_models.py b/tests/test_models.py
