SEAL 3.7.0 (OpenMined#319)

* SEAL 3.7.0

* bbreak

* SEAL 3.7

* master -> main

* update version minor

* debug long paths - windows

* disable windows for now

* disable windows for now

* add pytest slow mark

* pytest slow flag

* re-enable windows flows

* debug long paths for windows

* long paths for windows

* HEXL off

* enable HEXL only for unix

* bump version

.github/workflows/bazel_tests.yml

@@ -2,7 +2,7 @@ name: Bazel Tests
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
     types: [opened, synchronize, reopened]
 
@@ -28,5 +28,5 @@ jobs:
         timeout-minutes: 30
         run: bazel test --test_output=all --spawn_strategy=standalone --test_timeout=1500 --jobs 1 //tests/python/sealapi/...
       - name: Run TenSEAL tests
-        timeout-minutes: 60
-        run: bazel test --test_output=all --spawn_strategy=standalone --test_output=streamed --local_sigkill_grace_seconds=30 --test_timeout=2500 //tests/python/tenseal/...
+        timeout-minutes: 120
+        run: bazel test --test_output=all --spawn_strategy=standalone --test_output=streamed --local_sigkill_grace_seconds=30 --test_timeout=4000 //tests/python/tenseal/...

.github/workflows/benchmarks.yml

@@ -1,7 +1,7 @@
 name: Benchmarks
 on:
   push:
-      branches: [master]
+      branches: [main]
 
 jobs:
   benchmark:

.github/workflows/codeql-analysis.yml

@@ -2,7 +2,7 @@ name: "CodeQL"
 
 on:
   push:
-    branches: [ master ]
+    branches: [ main ]
 
 jobs:
   analyze:

.github/workflows/docker-image.yml

@@ -3,7 +3,7 @@ name: Docker Image
 on:
   push:
     branches:
-      - master
+      - main
   release:
     types: [created]
 

.github/workflows/ossar-analysis.yml

@@ -5,7 +5,7 @@ name: OSSAR
 
 on:
   push:
-    branches: [ master ]
+    branches: [ main ]
 
 jobs:
   OSSAR-Scan:

.github/workflows/tests.yml

@@ -2,7 +2,7 @@ name: Tests
 
 on:
   push:
-    branches: [master]
+    branches: [main]
   pull_request:
     types: [opened, synchronize, reopened]
 
@@ -44,6 +44,11 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Check LongPathsEnabled
+        run: |
+            git config --system core.longpaths true
+            (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled
+        if: ${{ matrix.os == 'windows-latest' }}
       - name: Prepare ENV
         run: |
             echo "CC=clang" >> $GITHUB_ENV
@@ -68,7 +73,7 @@ jobs:
         run: .github/workflows/scripts/build_windows.bat
         if: ${{ matrix.os == 'windows-latest' }}
       - name: Test with pytest
-        run: pytest -v ./tests/python/tenseal
+        run: pytest -m "not slow" -v ./tests/python/tenseal
       - name: Test with gtest for Ubuntu/MacOS
         run: .github/workflows/scripts/run_gtest_nix.sh
         if: ${{ matrix.os != 'windows-latest' }}
@@ -96,6 +101,11 @@ jobs:
             echo "CC=clang" >> $GITHUB_ENV
             echo "CXX=clang++" >> $GITHUB_ENV
         if: ${{ matrix.os == 'ubuntu-latest' }}
+      - name: Check LongPathsEnabled
+        run: |
+            git config --system core.longpaths true
+            (Get-ItemProperty "HKLM:System\CurrentControlSet\Control\FileSystem").LongPathsEnabled
+        if: ${{ matrix.os == 'windows-latest' }}
       - name: Install dependencies Ubuntu
         run: .github/workflows/scripts/install_req_ubuntu.sh
         if: ${{ matrix.os == 'ubuntu-latest' }}

cmake/seal.cmake

@@ -7,7 +7,7 @@ set(SEAL_USE_INTEL_HEXL ON)
 FetchContent_Declare(
   com_microsoft_seal
   GIT_REPOSITORY https://github.com/microsoft/SEAL
-  GIT_TAG        97e4b8ddbc1105fe8338f8c18ff3cafcba686266 #v3.6.5
+  GIT_TAG        1dd84a2066c1b12d56bbcec3e806a4d0653f2ca5 #v3.7
 )
 FetchContent_MakeAvailable(com_microsoft_seal)
 

pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+markers =
+    slow: marks tests as slow (deselect with '-m "not slow"')

setup.py

@@ -47,7 +47,9 @@ def run(self):
 
     def build_extension(self, ext):
         extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
+        hexl = "OFF" if platform.system() == "Windows" else "ON"
         cmake_args = [
+            "-DSEAL_USE_INTEL_HEXL=" + hexl,
             "-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=" + extdir,
             "-DPYTHON_EXECUTABLE=" + sys.executable,
         ]

tenseal/deps.bzl

@@ -26,9 +26,9 @@ def tenseal_deps():
         http_archive(
             name = "com_microsoft_seal",
             build_file = "//third_party:seal.BUILD",
-            strip_prefix = "SEAL-3.6.5",
-            sha256 = "77bfcb4a8b785206c419cdf7aff8c200250691518eeddc958f874d1f567b2872",
-            urls = ["https://github.com/microsoft/SEAL/archive/v3.6.5.tar.gz"],
+            strip_prefix = "SEAL-3.7.0",
+            sha256 = "d500e37df21eed97c638c00af6de3c9be6e1bc37dbc32ef8bce79b4cf72d18dd",
+            urls = ["https://github.com/microsoft/SEAL/archive/v3.7.0.tar.gz"],
         )
 
     if "com_xtensorstack_xtensor" not in native.existing_rules():

tenseal/version.py

@@ -1 +1 @@
-__version__ = "0.3.4"
+__version__ = "0.3.5"

tests/python/sealapi/test_sanity.py

@@ -24,7 +24,7 @@ def test_serialization_sanity():
     assert header.magic == 0xA15E
     assert header.header_size == 0x10
     assert header.version_major == 3
-    assert header.version_minor == 0x6
+    assert header.version_minor == 0x7
     assert header.compr_mode == sealapi.COMPR_MODE_TYPE.NONE
     assert header.size == 0
     assert header.reserved == 0

tests/python/tenseal/main.py

@@ -5,4 +5,8 @@
 import pytest
 
 if __name__ == "__main__":
-    sys.exit(pytest.main(["-s", "-v", "-x", os.path.dirname(os.path.abspath(__file__))]))
+    sys.exit(
+        pytest.main(
+            ["-m", "not slow", "-s", "-v", "-x", os.path.dirname(os.path.abspath(__file__))]
+        )
+    )

tests/python/tenseal/tensors/test_bfv_tensor.py

@@ -351,6 +351,7 @@ def test_add_sub_mul_tensor_ct_pt(context, shape, plain, op, reshape_first):
 @pytest.mark.parametrize("shape", BROADCAST_SHAPES)
 @pytest.mark.parametrize("plain", [True, False])
 @pytest.mark.parametrize("op", ["add", "sub", "mul"])
+@pytest.mark.slow
 def test_broadcast_add_sub_mul_tensor_ct_pt(context, shape, plain, op):
     l_t = np.random.randint(0, 100, *[shape[1]], dtype=np.int64)
     r_t = np.random.randint(0, 100, *[shape[0]], dtype=np.int64)
@@ -416,6 +417,7 @@ def test_broadcast_add_sub_mul_tensor_ct_pt(context, shape, plain, op):
 
 @pytest.mark.parametrize("shape", SHAPES)
 @pytest.mark.parametrize("op", ["add", "sub", "mul"])
+@pytest.mark.slow
 def test_add_sub_mul_scalar(context, shape, op):
     r_t = np.random.randint(0, 100, *[shape], dtype=np.int64)
     r_pt = ts.plain_tensor(r_t.flatten().tolist(), shape, dtype="int")
@@ -512,6 +514,7 @@ def test_power_inplace(context, plain, power, precision):
     ],
 )
 @pytest.mark.parametrize("reshape_first", [False, True])
+@pytest.mark.slow
 def test_polynomial(context, data, polynom, reshape_first):
     context = ts.context(ts.SCHEME_TYPE.BFV, 8192, 1032193)
 
@@ -571,6 +574,7 @@ def test_polynomial(context, data, polynom, reshape_first):
     ],
 )
 @pytest.mark.parametrize("plain", [True, False])
+@pytest.mark.slow
 def test_dot(context, shapes, plain):
     r_shape = shapes[0]
     l_shape = shapes[1]
@@ -621,10 +625,11 @@ def test_dot(context, shapes, plain):
     assert np.allclose(left_result, l_t, rtol=0, atol=0)
 
 
-# this test only test the python API for matmul
-# a previous test already test different shapes
+# this test only tests the python API for matmul
+# a previous test already tests different shapes
 @pytest.mark.parametrize("plain", [True, False])
 @pytest.mark.parametrize("arithmetic", [True, False])
+@pytest.mark.slow
 def test_matmul_api(context, plain, arithmetic):
     r_t = np.random.randint(0, 100, size=(2, 2), dtype=np.int64)
     l_t = np.random.randint(0, 100, size=(2, 2), dtype=np.int64)

tests/python/tenseal/tensors/test_ckks_tensor.py

@@ -328,6 +328,7 @@ def test_square_inplace(context, plain, precision):
 @pytest.mark.parametrize("plain", [True, False])
 @pytest.mark.parametrize("op", ["add", "sub", "mul"])
 @pytest.mark.parametrize("reshape_first", [False, True])
+@pytest.mark.slow
 def test_add_sub_mul_tensor_ct_pt(context, shape, plain, op, reshape_first):
     r_t = np.random.randn(*shape)
     l_t = np.random.randn(*shape)
@@ -403,6 +404,7 @@ def test_add_sub_mul_tensor_ct_pt(context, shape, plain, op, reshape_first):
 @pytest.mark.parametrize("shape", BROADCAST_SHAPES)
 @pytest.mark.parametrize("plain", [True, False])
 @pytest.mark.parametrize("op", ["add", "sub", "mul"])
+@pytest.mark.slow
 def test_broadcast_add_sub_mul_tensor_ct_pt(context, shape, plain, op):
     l_t = np.random.randn(*shape[1])
     r_t = np.random.randn(*shape[0])
@@ -468,6 +470,7 @@ def test_broadcast_add_sub_mul_tensor_ct_pt(context, shape, plain, op):
 
 @pytest.mark.parametrize("shape", SHAPES)
 @pytest.mark.parametrize("op", ["add", "sub", "mul"])
+@pytest.mark.slow
 def test_add_sub_mul_scalar(context, shape, op):
     r_t = np.random.randn(*shape)
     r_pt = ts.plain_tensor(r_t.flatten().tolist(), shape)
@@ -705,10 +708,11 @@ def test_dot(context, shapes, plain):
     assert np.allclose(left_result, l_t, rtol=0, atol=0.01)
 
 
-# this test only test the python API for matmul
-# a previous test already test different shapes
+# this test only tests the python API for matmul
+# a previous test already tests different shapes
 @pytest.mark.parametrize("plain", [True, False])
 @pytest.mark.parametrize("arithmetic", [True, False])
+@pytest.mark.slow
 def test_matmul_api(context, plain, arithmetic):
     r_t = np.random.randn(2, 2)
     l_t = np.random.randn(2, 2)

third_party/seal.BUILD

@@ -23,8 +23,8 @@ cmake_external(
         "make",
         "make install",
     ],
-    out_include_dir = "include/SEAL-3.6",
-    static_libraries = ["libseal-3.6.a"],
+    out_include_dir = "include/SEAL-3.7",
+    static_libraries = ["libseal-3.7.a"],
     visibility = ["//visibility:public"],
 )
 
@@ -44,8 +44,8 @@ cmake_external(
     install_prefix = "native/src",
     lib_source = ":src",
     make_commands = ["make"],
-    out_include_dir = "include/SEAL-3.6",
-    static_libraries = ["libseal-3.6.a"],
+    out_include_dir = "include/SEAL-3.7",
+    static_libraries = ["libseal-3.7.a"],
     visibility = ["//visibility:public"],
 )