CP-8874 Script to build the VSDK project with various commands

.bumpversion.cfg

@@ -3,16 +3,17 @@ current_version = 4.0.6.dev0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<dev>\d+))?
-serialize = 
+serialize =
 	{major}.{minor}.{patch}.{release}{dev}
 	{major}.{minor}.{patch}
 
 [bumpversion:part:release]
-optional_value = prod
+optional_value = _
 first_value = dev
-values = 
+values =
 	dev
-	prod
+	rc
+	_
 
 [bumpversion:part:dev]
 
@@ -26,3 +27,7 @@ values =
 
 [bumpversion:file:./tools/src/main/python/dlpx/virtualization/_internal/VERSION]
 
+[bumpversion:file:./tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py]
+search = DVP_VERSION = '{current_version}'
+replace = DVP_VERSION = '{new_version}'
+

.github/workflows/pre-commit.yml

@@ -1,158 +1,66 @@
-name: Pre-commit actions for Delphix Virtualization SDK
+#
+# Copyright (c) 2020, 2022 by Delphix. All rights reserved.
+#
 
-on: [pull_request]
+name: "Pre-commit actions for Delphix Virtualization SDK"
 
-jobs:
-  pytest27:
-    name: Test ${{ matrix.package }} on ${{ matrix.os }} using pytest (Python 2.7)
-    runs-on: ${{ matrix.os }}
-    strategy:
-      max-parallel: 4
-      matrix:
-        python-version: [2.7]
-        os: [ubuntu-latest, macos-10.15, windows-latest]
-        package: [common, libs, platform]
-
-    steps:
-    - name: Checkout ${{ matrix.package }} project
-      uses: actions/checkout@v3
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install ${{ matrix.package }} dependencies
-      working-directory: ${{ matrix.package }}
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt --find-links https://test.pypi.org/simple/dvp-api/
-
-    - name: Install ${{ matrix.package }} project
-      working-directory: ${{ matrix.package }}
-      run: |
-        pip install . --find-links https://test.pypi.org/simple/dvp-api/
-
-    - name: Test ${{ matrix.package }} project with pytest
-      working-directory: ${{ matrix.package }}
-      run: |
-        python -m pytest src/test/python
+on:
+  # It is important to check the commits done to master, develop and release branches only.
+  pull_request:
+    branches:
+      - master
+      - develop
+      - release
 
+jobs:
   pytest38:
     name: Test ${{ matrix.package }} on ${{ matrix.os }} using pytest (Python 3.8)
     runs-on: ${{ matrix.os }}
     strategy:
       max-parallel: 4
       matrix:
-        python-version: [3.8]
-        os: [ubuntu-latest, macos-latest, windows-latest]
-        package: [common, libs, platform, tools]
-
-    steps:
-    - name: Checkout ${{ matrix.package }} project
-      uses: actions/checkout@v1
-
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install ${{ matrix.package }} dependencies
-      working-directory: ${{ matrix.package }}
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements.txt --find-links https://test.pypi.org/simple/dvp-api/
-
-    - name: Install ${{ matrix.package }} project
-      working-directory: ${{ matrix.package }}
-      run: |
-        pip install . --find-links https://test.pypi.org/simple/dvp-api/
-
-    - name: Test ${{ matrix.package }} project with pytest
-      working-directory: ${{ matrix.package }}
-      run: |
-        python -m pytest src/test/python
-
-  lintpython27:
-    name: Lint ${{ matrix.package }} - Python27
-
-    runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        package: [common, libs, platform]
+        python-version: [ 3.8 ]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        package: [ common, libs, platform, tools ]
 
     steps:
-      - name: Checkout ${{ matrix.package }}
-        uses: actions/checkout@v1
+      - name: Checkout ${{ matrix.package }} project
+        uses: actions/checkout@v3
 
-      - name: Set up Python 2.7
-        uses: actions/setup-python@v1
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
         with:
-          python-version: 2.7
+          python-version: ${{ matrix.python-version }}
 
-      - name: Install flake8
+      - name: Install ${{ matrix.package }} dependencies
+        working-directory: ${{ matrix.package }}
         run: |
           python -m pip install --upgrade pip
-          pip install flake8
+          pip install -r requirements.txt --find-links https://test.pypi.org/simple/dvp-api/
 
-      - name: Run flake8 on src directory
+      - name: Install ${{ matrix.package }} project
         working-directory: ${{ matrix.package }}
-        run: python -m flake8 src/main/python --max-line-length 88
+        run: |
+          pip install . --find-links https://test.pypi.org/simple/dvp-api/
 
-      - name: Run flake8 on test directory
+      # Run all the test cases part of the package.
+      - name: Test ${{ matrix.package }} project with pytest
         working-directory: ${{ matrix.package }}
-        run: python -m flake8 src/test/python --max-line-length 88
-
-  lintpython38:
-    name: Lint ${{ matrix.package }} - Python38
-
-    runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 4
-      matrix:
-        package: [common, libs, platform, tools]
-
-    steps:
-      - name: Checkout ${{ matrix.package }}
-        uses: actions/checkout@v1
-
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.8
+        run: |
+          python -m pytest src/test/python
 
+      # Install flake8 and run linting on src and test for linting if OS is ubuntu.
       - name: Install flake8
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         run: |
-          python -m pip install --upgrade pip
           pip install flake8
 
       - name: Run flake8 on src directory
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         working-directory: ${{ matrix.package }}
         run: python -m flake8 src/main/python --max-line-length 88
 
       - name: Run flake8 on test directory
+        if: ${{ matrix.os == 'ubuntu-latest' }}
         working-directory: ${{ matrix.package }}
         run: python -m flake8 src/test/python --max-line-length 88
-
-  #format:
-    #name: Check format ${{ matrix.package}}
-
-    #runs-on: ubuntu-latest
-    #strategy:
-      #max-parallel: 4
-      #matrix:
-        #package: [common, libs, platform, tools]
-
-    #steps:
-      #- uses: actions/checkout@v1
-
-      #- name: Check src format
-        #uses: lgeiger/[email protected]
-        #with:
-          #args: "${{ matrix.package }}/src/main/python -t py27 --check"
-
-      #- name: Check test format
-        #uses: lgeiger/[email protected]
-        #with:
-          #args: "${{ matrix.package }}/src/test/python -t py27 --check"

.github/workflows/publish-docs.yml

@@ -1,3 +1,7 @@
+#
+# Copyright (c) 2020, 2022 by Delphix. All rights reserved.
+#
+
 name: Publish docs to GitHub pages
 
 on:
@@ -16,42 +20,52 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7]
-        package: [docs]
-        repository: ['delphix/virtualization-sdk']
+        python-version: [ 3.7 ]
+        package: [ docs ]
+        repository: [ 'delphix/virtualization-sdk' ]
 
     steps:
+      ###
+      # Only a single commit is fetched by default, for the ref/SHA that triggered the workflow.
+      # Set fetch-depth: 0 to fetch all history for all branches and tags.
+      ###
       - uses: actions/checkout@v2
         with:
-          # Only a single commit is fetched by default, for the ref/SHA that triggered the workflow.
-          # Set fetch-depth: 0 to fetch all history for all branches and tags.
           fetch-depth: 0
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
+
+      ###
+      # Set CURRENT_BRANCH environment variable to the current branch name. Refrain from using 'set-env' as
+      # GitHub has identified the command as a moderate security vulnerability.
+      ###
       - name: Set the current branch environment variable
         working-directory: ${{ matrix.package }}
         run: |
           CURRENT_BRANCH_VAR=${GITHUB_REF#refs/heads/}
           echo "Current branch: $CURRENT_BRANCH_VAR"
-          # Set CURRENT_BRANCH environment variable to the current branch name. Refrain from using 'set-env' as
-          # GitHub has identified the command as a moderate security vulnerability.
           echo "CURRENT_BRANCH=$(echo $CURRENT_BRANCH_VAR)" >> $GITHUB_ENV
+
+      # Display all remote branches
       - name: Display all remote branches
         working-directory: ${{ matrix.package }}
-        run: |
-          # Display all remote branches
-          git branch -r
+        run: git branch -r
+
+      ###
+      # Get only docs branches, extract the "docs/x.y.z" part, sort them in descending order, and get the first one.
+      # Set the LATEST_DOCS_BRANCH environment variable. Refrain from using 'set-env' as GitHub has identified the
+      # command as a moderate security vulnerability.
+      ###
       - name: Set the latest docs branch environment variable
         working-directory: ${{ matrix.package }}
         run: |
-          # Get only docs branches, extract the "docs/x.y.z" part, sort them in descending order, and get the first one.
           LATEST_DOCS_BRANCH_VAR=$(git branch -r | grep -e ".*\/*docs\/[0-9].[0-9].[0-9]" | sed -n "s/.*\/*\(docs\/[0-9].[0-9].[0-9]\).*/\1/p" | sort -r | head -n 1)
           echo "Latest docs branch: $LATEST_DOCS_BRANCH_VAR"
-          # Set the LATEST_DOCS_BRANCH environment variable. Refrain from using 'set-env' as GitHub has identified the
-          # command as a moderate security vulnerability.
           echo "LATEST_DOCS_BRANCH=$(echo $LATEST_DOCS_BRANCH_VAR)" >> $GITHUB_ENV
+
       - name: Check that the current branch is the latest docs branch, fail otherwise
         working-directory: ${{ matrix.package }}
         run: |
@@ -61,16 +75,19 @@ jobs:
           else
             echo "The action is running on the latest docs branch."
           fi
+
       - name: Install Pipenv
         uses: dschep/install-pipenv-action@v1
+
       - name: Install dependencies for building documenation
         working-directory: ${{ matrix.package }}
-        run: |
-          pipenv install
+        run: pipenv install
+
       - name: Build documentation
         working-directory: ${{ matrix.package }}
-        run: |
-          pipenv run mkdocs build --clean
+        run: pipenv run mkdocs build --clean
+
+      # Docs will be pushed to developer.delphix.com if the repository is delphix/virtualization-sdk.
       - name: Deploy the contents of docs/site to gh-pages (developer.delphix.com) 🚀
         if: ${{ github.repository == matrix.repository }}
         uses: peaceiris/actions-gh-pages@v3
@@ -81,6 +98,7 @@ jobs:
           user_name: "github-actions[bot]"
           user_email: "github-actions[bot]@users.noreply.github.com"
           cname: developer.delphix.com
+
       - name: Deploy the contents of docs/site to personal gh-pages 🚀
         if: ${{ github.repository != matrix.repository }}
         uses: peaceiris/actions-gh-pages@v3

README-dev.md

@@ -1,4 +1,4 @@
-# Copyright (c) 2019 by Delphix. All rights reserved.
+# Copyright (c) 2019, 2022 by Delphix. All rights reserved.
 
 # Delphix Virtualization SDK
 
@@ -63,9 +63,8 @@ To install the SDK, follow these steps:
    One of the SDK dependencies - dvp-api - is currently hosted on [TestPyPi](https://test.pypi.org/project/dvp-api/). 
    By default `pip` looks at pypi.org for packages to install. In order to successfully install the SDK, you have to 
    configure pip to search an additional package repository - test.pypi.org.
-2. Go into one of the package directories (common, dvp, libs, platform, tools) and run the commands below.
-3. Install the package's development dependencies: `pip install -r requirements.txt`.
-4. Install the package itself (use `-e` flag if you want to install the package in editable mode): `pip install .`.
+2. To build the project, run `sh bin/build_project.sh -b`.
+3. For more information on the script options, use `sh bin/build_project.sh -h`.
 
 
 ### CLI changes
@@ -111,9 +110,8 @@ Go into one of the package directories (common, dvp, libs, platform, tools) and
 
 1. Install the package's development dependencies and package itself by following the directions in [the SDK installation section](#installing-the-sdk-from-source).
 2. Run unit tests: `python -m pytest src/test/python`.
-
-There's no way to locally run unit tests in all packages with one command. However, they will be run automatically
-through GitHub Actions when you open a pull request. You can always open a draft pull request 
+3. To run unit tests in all packages with one command, run `sh bin/build_project.sh -t`.
+4. For more information on the script options, use `sh bin/build_project.sh -h`.
 
 ### Manual testing