Fully functional bundled Kaolin-Dash3d visualizer for meshes and point clouds

.drone.yml

@@ -7,6 +7,7 @@ environment:
   PYTORCH_VERSION: "1.6"
   CUDNN_VERSION: "7"
   TORCH_CUDA_ARCH_LIST: "5.2 6.0 6.1 7.0 7.5+PTX"
+  KAOLIN_INSTALL_EXPERIMENTAL: "1"
 
 steps:
 - name: submodules
@@ -42,6 +43,11 @@ steps:
   - export PATH=/opt/conda/bin:$PATH
   - pip install --upgrade pip>=19.3
   - conda install -y pytorch==1.6 cudatoolkit=10.1 -c pytorch
+  - conda install -c conda-forge nodejs
+  - >
+    apt-get update && apt-get install -y --no-install-recommends
+    libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
+  - npm install
   - python setup.py develop
   - pip install -r tools/ci_requirements.txt
-  - pytest --cov=kaolin/ tests/python && pytest --doctest-modules kaolin/
+  - pytest --cov=kaolin/ tests/python tests/integration && pytest --doctest-modules kaolin/

.gitignore

@@ -15,9 +15,14 @@ kaolin.egg-info
 *.egg
 .eggs/
 kaolin/cython/**/*.cpp
+node_modules/
 
 # byte-compiled python files
 *.py[cod]
 
 # pip cache
 .cache
+
+# editor specific
+*~
+.idea

CONTRIBUTING.md

@@ -47,7 +47,8 @@ Include a license at the top of new files.
 * [Python example](examples_license.py)
 
 #### Code organization
-* [kaolin](kaolin/) - The Core of Kaolin library, everything that is not in [csrc](kaolin/csrc) is a Python module.
+* [kaolin](kaolin/) - The core Kaolin library, comprised of python modules, 
+except for code under [csrc](kaolin/csrc) or [experimental](kaolin/experimental).
   * [csrc](kaolin/csrc/) - Directory for all the C++ / CUDA implementations of custom ops.
     The gpu ops parts will be under the subdirectory [csrc/cuda](kaolin/csrc/cuda)
     while the cpu parts will be under the subdirectory [csrc/cpu](kaolin/csrc/cpu).
@@ -57,6 +58,7 @@ Include a license at the top of new files.
   * [render](kaolin/render/) - Module of all the differentiable renderers modules and advanced implementations.
   * [utils](kaolin/utils/) - Module of all the utility features for debugging and testing.
   * [visualize](kaolin/visualize/) - Module of all the visualization modules.
+  * [experimental](kaolin/experimental/) - Contains less thoroughly tested components for early adoption.
 * [examples](examples/) - Examples of Kaolin usage
 * [tests](tests/) - Tests for all Kaolin
 
@@ -70,16 +72,18 @@ We follow [PEP8 Style Guide](https://www.python.org/dev/peps/pep-0008/) with som
 
 It is enforced using [flake8](https://pypi.org/project/flake8/), with [flake8-bugbear](https://pypi.org/project/flake8-bugbear/), [flake8-comprehensions](https://pypi.org/project/flake8-comprehensions/), [flake8-mypy](https://pypi.org/project/flake8-mypy/) and [flake8-pyi](https://pypi.org/project/flake8-pyi/)
 
-to run flake8 execute ``flake8 --config=.flake8 .`` from the [root of kaolin](https://gitlab-master.nvidia.com/Toronto_DL_Lab/kaolin-reformat)
+To run flake8 execute ``flake8 --config=.flake8 .`` from the [root of kaolin](https://gitlab-master.nvidia.com/Toronto_DL_Lab/kaolin-reformat).
 
 On top of that we use prefixes (``packed\_``, ``padded\_``) to indicate that a module / op is specific to a layout, an , all ops of the same purpose for different layouts should be in the same file.
 
 [tests/python/kaolin/](tests/python/kaolin) should follows the same directory structure of [kaolin/](kaolin/). E.g. each module kaolin/path/to/mymodule.py should have a corresponding tests/python/kaolin/path/to/test\_mymodule.py.
 
 #### Tests
 We are applying [pytest](https://docs.pytest.org/en/latest/) on [tests/python directory](tests/python/), with [pytest-cov](https://pytest-cov.readthedocs.io/en/latest/), which should follows the directory structure of [kaolin](kaolin/).
+To run these tests execute ``pytest --cov=kaolin/ tests/`` from the [root of kaolin](https://gitlab-master.nvidia.com/Toronto_DL_Lab/kaolin-reformat).
 
-to run the tests execute ``pytest --cov=kaolin/ tests/`` from the [root of kaolin](https://gitlab-master.nvidia.com/Toronto_DL_Lab/kaolin-reformat)
+Additional integration tests are located in [tests/integration](tests/integration).
+See that subdirectory for details.
 
 #### Documentation
 Contributors are encouraged to verify the generated documentation before each pull request.

MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include kaolin/experimental/dash3d/static *
+recursive-include kaolin/experimental/dash3d/templates *

ci/pytest-linux.sh

@@ -16,4 +16,4 @@ docker run \
     --runtime=nvidia \
     --rm \
     "$IMAGE_NAME" \
-    /bin/bash -c "pytest --cov=kaolin/ tests/python && pytest --doctest-modules kaolin/"
+    /bin/bash -c "pytest --cov=kaolin/ tests/python tests/integration && pytest --doctest-modules kaolin/"

docs/notes/installation.rst

@@ -39,12 +39,16 @@ To install the library. You must first clone the repository:
     $ git clone --recursive https://github.com/NVIDIAGameWorks/kaolin
     $ cd kaolin
 
-you can then select the tag if you want 0.9.0 or 0.1 specific release, example:
+You can then select the tag if you want 0.9.0 or 0.1 specific release, example:
 
 .. code-block:: bash
 
     $ git checkout v0.9.0
 
+To enable installation of experimental features, set
+environment variable
+``export KAOLIN_INSTALL_EXPERIMENTAL=1``. To install, run:
+
 .. code-block:: bash
 
     $ python setup.py develop

examples/tutorial/visualize_main.py

@@ -107,10 +107,14 @@ def __normalize_vertices(vertices):
             # Emulate a training update
             out_pt_clouds = []
             out_vert_list = []
+            out_voxels = []
             for i in range(len(gt_vert_list)):
                 delta_weight = 1.0 - iteration / (args.iterations - 1)
                 out_vert_list.append(gt_vert_list[i] * (1.0 + delta_list[i] * delta_weight))
                 out_pt_clouds.append(input_pt_clouds[i] * (1.0 + delta_pt_list[i] * delta_weight))
+                vg = kaolin.ops.conversions.trianglemeshes_to_voxelgrids(
+                    out_vert_list[-1].unsqueeze(0), face_list[i], 30)
+                out_voxels.append(vg.squeeze(0).bool())
 
             # Save model predictions to track training progress over time
             timelapse.add_mesh_batch(
@@ -122,10 +126,14 @@ def __normalize_vertices(vertices):
                 iteration=iteration,
                 category='output',
                 pointcloud_list=out_pt_clouds)
+            timelapse.add_voxelgrid_batch(
+                iteration=iteration,
+                category='output',
+                voxelgrid_list=out_voxels)
 
     logger.info('Emulated training complete!\n'
                 'You can now view created USD files found here: {}\n\n'
-                'You will soon be able to visualize these in the Kaolin Research Omniverse App '
+                'You will soon be able to visualize these in the Kaolin Omniverse App '
                 'and our web visualizer. Stay tuned!'.format(args.output_dir))
 
     # TODO(mshugrina): update command line once finalized

kaolin/experimental/dash3d/Makefile

@@ -0,0 +1,52 @@
+CORE_JS = $(wildcard src/*.js)
+CORE_CSS = $(wildcard src/*.css)
+THIRDPARTY_JS = ../../../node_modules/jquery/dist/jquery.min.js \
+    ../../../node_modules/three/build/three.min.js \
+    ../../../node_modules/three/examples/js/controls/OrbitControls.js   # TODO: this needs to be minified
+THIRDPARTY_CSS = ../../../node_modules/mustard-ui/dist/css/mustard-ui.min.css
+JSMIN=uglifyjs
+AWK=awk
+
+all: css js
+
+js: jscore jsthirdparty
+
+css: csscore cssthirdparty
+
+info: info_jscore info_jsthirdparty info_csscore info_cssthirdparty
+
+info_jscore: $(CORE_JS)
+	echo "core js files: " $(CORE_JS)
+
+info_jsthirdparty: $(THIRDPARTY_JS)
+	echo "thirdparty js files: " $(THIRDPARTY_JS)
+
+info_csscore: $(CORE_CSS)
+	echo "core css files: " $(CORE_CSS)
+
+info_cssthirdparty: $(THIRDPARTY_CSS)
+	echo "thirdparty css files: " $(THIRDPARTY_CSS)
+
+jscore: static/core-min.js
+
+static/core-min.js: $(CORE_JS)
+	cat src/copyright.js > static/core-min.js
+	$(JSMIN) $^ -m >> static/core-min.js
+
+jsthirdparty: static/thirdparty.js
+
+static/thirdparty.js: $(THIRDPARTY_JS)
+	$(AWK) 1 $^ > static/thirdparty.js
+
+csscore: static/style.css
+
+static/style.css: $(CORE_CSS)
+	cat $^ > static/style.css
+
+cssthirdparty: static/thirdparty.css
+
+static/thirdparty.css: $(THIRDPARTY_CSS)
+	cat $^ > static/thirdparty.css
+
+clean:
+	rm -f static/style.css static/core-min.js static/thirdparty.js static/thirdparty.css

kaolin/experimental/dash3d/README.md

@@ -0,0 +1,81 @@
+
+# Kaolin Dash3D (beta)
+
+WebGL dashboard that allows viewing checkpoints written
+with kaolin's `Timelapse` from any remote machine in a web browser. 
+Kaolin Dash3D is a **lightweight alternative** to the Training Visualizer
+within the 
+[Kaolin Omniverse App](https://docs.omniverse.nvidia.com/app_kaolin/app_kaolin/user_manual.html),
+which is a desktop application with wider visualization support
+and superior rendering quality. 
+
+Currently Dash3D can only display geometric data of the following types:
+
+* 3d meshes (no texture or vertex color support)
+* point clouds (no color)
+
+Voxelgrid, as well as texture and color support is planned.
+
+
+## Using
+
+Simply run `kaolin-dash3d` on the machine that has your training checkpoints:
+
+```
+$ kaolin-dash3d --logdir=$MY_EXPERIMENTS/test0/checkpts --port=8080
+```
+
+Then, view the 3D checkpoints in a browser window locally by going to
+[localhost:8080](http://localhost:8080/), or connect to the server remotely.
+(Tip: use ssh port forwarding on your client machine.)
+
+## Bundled Third-party code
+
+We would like to acknowledge the following third-party code bundled with
+this application:
+* [jQuery](https://jquery.com/download/), released under [MIT License](https://jquery.org/license/) 
+* [ThreeJS](https://github.com/mrdoob/three.js/), released under [MIT License](https://github.com/mrdoob/three.js/blob/dev/LICENSE)
+* [Mustard UI](https://kylelogue.github.io/mustard-ui/index.html), released under [MIT License](https://github.com/kylelogue/mustard-ui/blob/master/LICENSE)
+
+## Developing
+
+#### Dependencies
+
+Third party dependencies for the javascript frontend as well as for 
+testing are managed by `npm`. To fetch dependencies, first 
+install `nodejs`, which is available through `conda`, and run `npm install`
+from the root of kaolin (this will install both production and development
+dependencies). 
+
+To add new dependencies, run this from the root of kaolin:
+```
+npm install <PKG> --save-dev
+```
+This will also update the `package.json` file in the root of kaolin.
+If the new dependencies must be served to the client, omit 
+`--save-dev` from the command, and add the appropriate source to
+the `Makefile` in this directory. 
+
+#### Compiling client side code
+
+Compiled source and thirdparty javascript is bundled with the app in the 
+`static` subdirectory, while source javascript and CSS code is
+located under `src/`. After edits, it
+must be compiled into an optimized version to be served:
+```
+$ cd kaolin/visualize/dash3d
+$ make clean
+$ make
+```
+Note that this also compiles thirdparty dependencies which are
+assumed to reside in `node-modules` in the root of `kaolin`, a 
+directory created automatically by `npm`. This is not ideal,
+but for now we avoid setting up tools like WebPack, 
+given our small set of dependencies.
+
+#### Integration testing
+
+Integration tests are located in `kaolin/tests/integration`. Currently
+the set of implemented tests is limited and excluded from continuous 
+integration due to their complexity. Please refer to the README
+in the test directory.

kaolin/experimental/dash3d/kaolin-dash3d

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from kaolin.experimental.dash3d.run import run_main
+
+if __name__ == "__main__":
+    run_main()

kaolin/experimental/dash3d/run.py

@@ -0,0 +1,86 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import print_function
+
+import argparse
+import logging
+import os
+import sys
+
+from flask import Flask, render_template
+from tornado.wsgi import WSGIContainer
+from tornado.web import Application, FallbackHandler
+from tornado.ioloop import IOLoop
+
+from kaolin.experimental.dash3d.util import StreamingGeometryHelper, GeometryWebSocketHandler
+
+logger = logging.getLogger(__name__)
+
+
+def create_server(logdir):
+    """ Create the server, including websocket handler through tornado, and flask http server.
+
+    Args:
+        logdir (str): directory where Timelapse data is written.
+    """
+    # Helper for streaming geometry from the logdir
+    helper = StreamingGeometryHelper(logdir)
+
+    # Flask for HTTP
+    _base_dir = os.path.dirname(__file__)
+    _template_dir = os.path.join(_base_dir, 'templates')
+    _static_dir = os.path.join(_base_dir, 'static')
+    app = Flask('kaolin_dash3d',
+                template_folder=_template_dir,
+                static_url_path='/static',
+                static_folder=_static_dir)
+
+    @app.route('/')
+    def index():
+        helper.parser.check_for_updates()
+        return render_template('home.html', logdir=helper.logdir,
+                               nmeshes=helper.parser.num_mesh_categories(),
+                               npointclouds=helper.parser.num_pointcloud_categories())
+
+    # Tornado server to handle websockets
+    container = WSGIContainer(app)
+    server = Application([
+        (r'/websocket/', GeometryWebSocketHandler, dict(helper=helper)),
+        (r'.*', FallbackHandler, dict(fallback=container))
+    ])
+    return server
+
+
+def run_main():
+    aparser = argparse.ArgumentParser(
+        description='NVIDIA Kaolin Tensorboard 3d visualizer for USD files generated during training.')
+    aparser.add_argument('--logdir', action='store', type=str, required=True,
+                         help='The vis folder generated by the Timelapse module.')
+    aparser.add_argument('--log_level', action='store', type=int, default=logging.INFO,
+                         help='Logging level, DEBUG: 10, INFO: 20, WARN: 30, ERROR: 40.')
+    aparser.add_argument('--port', action='store', default=8080)
+    args = aparser.parse_args()
+
+    logging.basicConfig(level=args.log_level,
+                        format='%(asctime)s|%(levelname)8s|%(name)15s| %(message)s',
+                        handlers=[logging.StreamHandler(sys.stdout)])
+
+    print(f'Dash3D server starting. Go to: http://localhost:{args.port}')
+    server = create_server(args.logdir)
+    server.listen(args.port)
+    IOLoop.instance().start()
+
+
+if __name__ == "__main__":
+    run_main()