Enhance license checker to cover multiple license header and md files (…

…#18633)

LICENSE

@@ -216,7 +216,7 @@
     The following components are provided under an Apache 2.0 license.
 
     1. MXNet Cpp-package - For details, /cpp-package/LICENSE
-         Copyright (c) 2015-2016 by Contributors 
+         Copyright (c) 2015-2016 by Contributors
     2. MXNet rcnn - For details, see, example/rcnn/LICENSE
          Copyright (c) 2014, 2015, The Regents of the University of California (Regents)
     3. MXNet scala-package - For details, see, scala-package/LICENSE
@@ -226,9 +226,9 @@
     5. 3rdparty/dlpack - For details, see, 3rdparty/dlpack/LICENSE
          Copyright 2017 by Contributors
     6. 3rdparty/dmlc-core - For details, see, 3rdparty/dmlc-core/LICENSE
-         Copyright (c) 2015 by Contributors 
+         Copyright (c) 2015 by Contributors
          Copyright 2015 by dmlc-core developers
-         Copyright by Contributors 
+         Copyright by Contributors
     7. 3rdparty/mshadow - For details, see, 3rdparty/mshadow/LICENSE
          Copyright (c) 2014-2016 by Contributors
          Copyright by Contributors
@@ -237,7 +237,7 @@
          Copyright 2018 by Contributors
          Copyright (c) 2018 by Xilinx, Contributors
     9. 3rdparty/tvm/dmlc-core - For details, see, 3rdparty/tvm/3rdparty/dmlc-core/LICENSE
-         Copyright (c) 2015 by Contributors 
+         Copyright (c) 2015 by Contributors
     10. 3rdparty/tvm/dlpack - For details, see, 3rdparty/tvm/3rdparty/dlpack/LICENSE
          Copyright (c) 2015-2017 by Contributors
          Copyright by Contributors
@@ -343,6 +343,12 @@
     11. Numpy einsum operator - For details, see src/operator/numpy/np_einsum_op-inl.h
          Copyright (c) 2005-2019, NumPy Developers.
          Copyright (c) 2019, The Apache Software Foundation.
+    12. Numpy einsum operator - For details, see src/operator/numpy/np_einsum_path_op-inl.h
+         Copyright (c) 2005-2019, NumPy Developers.
+         Copyright (c) 2019, The Apache Software Foundation.
+    13. Numpy einsum operator - For details, see src/operator/numpy/np_einsum_op.cc
+         Copyright (c) 2005-2019, NumPy Developers.
+         Copyright (c) 2019, The Apache Software Foundation.
 
     =======================================================================================
     2-clause BSD licenses
@@ -385,14 +391,18 @@
     5. im2col.cuh - For details, see, src/operator/nn/im2col.cuh
          Copyright (c) 2014-2017 The Regents of the University of California (Regents)
          Copyright (c) 2014-2017, the respective contributors
-
     6. deformable_im2col.h - For details, see, src/operator/contrib/nn/deformable_im2col.h
          Copyright (c) 2014-2017 The Regents of the University of California (Regents)
          Copyright (c) 2014-2017, the respective contributors
-
     7. deformable_im2col.cuh - For details, see, src/operator/contrib/nn/deformable_im2col.cuh
          Copyright (c) 2014-2017 The Regents of the University of California (Regents)
          Copyright (c) 2014-2017, the respective contributors
+    8. modulated_deformable_im2col.h - For details, see, src/operator/contrib/nn/modulated_deformable_im2col.h
+         Copyright (c) 2014-2017 The Regents of the University of California (Regents)
+         Copyright (c) 2014-2017, the respective contributors
+    9. modulated_deformable_im2col.cuh - For details, see, src/operator/contrib/nn/modulated_deformable_im2col.cuh
+         Copyright (c) 2014-2017 The Regents of the University of California (Regents)
+         Copyright (c) 2014-2017, the respective contributors
 
 
     COPYRIGHT
@@ -667,7 +677,7 @@
     <none yet>
 
     =======================================================================================
-    
+
     13. FindJeMalloc.cmake
     For details, see cmake/Modules/FindJeMalloc.cmake
 
@@ -690,14 +700,14 @@
     # execute, and transmit the Software, and to prepare derivative works of the
     # Software, and to permit third-parties to whom the Software is furnished to
     # do so, all subject to the following:
-    # 
+    #
     # The copyright notices in the Software and this entire statement, including
     # the above license grant, this restriction and the following disclaimer,
     # must be included in all copies of the Software, in whole or in part, and
     # all derivative works of the Software, unless such copies or derivative
     # works are solely in the form of machine-executable object code generated by
     # a source language processor.
-    # 
+    #
     # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     # FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
@@ -706,7 +716,7 @@
     # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
     # DEALINGS IN THE SOFTWARE.
 
-    Now it has been changed by Apache MXNet project. Modifications have the following license: 
+    Now it has been changed by Apache MXNet project. Modifications have the following license:
 
     # Licensed to the Apache Software Foundation (ASF) under one
     # or more contributor license agreements.  See the NOTICE file
@@ -724,8 +734,8 @@
     # KIND, either express or implied.  See the License for the
     # specific language governing permissions and limitations
     # under the License.
-    
-    
+
+
     # Copyright (c)      2014 Thomas Heller
     # Copyright (c) 2007-2012 Hartmut Kaiser
     # Copyright (c) 2010-2011 Matt Anderson
@@ -848,28 +858,28 @@
     /*
      A C-program for MT19937, with initialization improved 2002/1/26.
      Coded by Takuji Nishimura and Makoto Matsumoto.
-    
+
      Before using, initialize the state by using init_genrand(seed)
      or init_by_array(init_key, key_length).
-    
+
      Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
      All rights reserved.
-    
+
      Redistribution and use in source and binary forms, with or without
      modification, are permitted provided that the following conditions
      are met:
-    
+
      1. Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
-    
+
      2. Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
-    
+
      3. The names of its contributors may not be used to endorse or promote
      products derived from this software without specific prior written
      permission.
-    
+
      THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
      "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
      LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -881,8 +891,8 @@
      LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
      NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
      SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-    
-    
+
+
      Any feedback is very welcome.
      http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html
      email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space)
@@ -922,28 +932,28 @@
     For details, see /3rdparty/tvm/3rdparty/rang/LICENSE
 
     This is free and unencumbered software released into the public domain.
-    
+
     Anyone is free to copy, modify, publish, use, compile, sell, or
     distribute this software, either in source code form or as a compiled
     binary, for any purpose, commercial or non-commercial, and by any
     means.
-    
+
     In jurisdictions that recognize copyright laws, the author or authors
     of this software dedicate any and all copyright interest in the
     software to the public domain. We make this dedication for the benefit
     of the public at large and to the detriment of our heirs and
     successors. We intend this dedication to be an overt act of
     relinquishment in perpetuity of all present and future rights to this
     software under copyright law.
-    
+
     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
     IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
     OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     OTHER DEALINGS IN THE SOFTWARE.
-    
+
     For more information, please refer to <http://unlicense.org>
 
     20. Font-Awesome, SIL Open Font License(OFL)

src/operator/numpy/np_einsum_op-inl.h

@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
 /*
  * Copyright (c) 2005-2019, NumPy Developers.
  * Copyright (c) 2019, The Apache Software Foundation.

tests/nightly/apache_rat_license_check/rat-excludes

@@ -88,5 +88,4 @@ LICENSE.binary.dependencies
 .*FindCUDAToolkit.cmake
 .*select_compute_arch.cmake
 image-classification-predict.cc
-np_einsum_op-inl.h
 DISCLAIMER-WIP

tests/python/unittest/onnx/README.md

@@ -1,3 +1,20 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+<!--- -->
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+<!--- -->
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
 # ONNX tests
 
 ## Directory structure:

tools/license_header.py

@@ -58,15 +58,27 @@
 under the License."""
 
 # if a file contains any str in the list, then consider it has been licensed
-_LICENSE_PATTERNS = ['Licensed to the Apache Software Foundation']
+_APACHE_LICENSE_PATTERNS = ['Licensed to the Apache Software Foundation']
+_OTHER_LICENSE_PATTERNS = ['THE SOFTWARE IS PROVIDED \"AS IS\"',
+                           'THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS']
+TOP_LEVEL_LICENSE_FILE = 'LICENSE'
 
 # the folders or files that will be ignored
 _WHITE_LIST = [
                # Licensed under docker/Dockerfiles/License.md
                'docker/Dockerfiles',
 
                # Git submodules under different licenses
-               '3rdparty',
+               '3rdparty/ctc_include/contrib/moderngpu',
+               '3rdparty/dlpack',
+               '3rdparty/dmlc-core',
+               '3rdparty/googletest',
+               '3rdparty/mkldnn',
+               '3rdparty/nvidia_cub',
+               '3rdparty/onnx-tensorrt',
+               '3rdparty/openmp',
+               '3rdparty/ps-lite',
+               '3rdparty/tvm',
 
                # 3rdparty headerfiles under different licenses
                'include/mkldnn',
@@ -96,15 +108,24 @@
                'docs/static_site/src/assets/js/clipboard.js',
                'cmake/upstream/FindCUDAToolkit.cmake',
                'cmake/upstream/select_compute_arch.cmake',
-               'src/operator/numpy/np_einsum_op-inl.h',
+
+               # This file
+               'tools/license_header.py',
+
+               # Github template
+               '.github/ISSUE_TEMPLATE/bug_report.md',
+               '.github/ISSUE_TEMPLATE/feature_request.md',
+               '.github/ISSUE_TEMPLATE/flaky_test.md',
+               '.github/PULL_REQUEST_TEMPLATE.md'
                ]
 
 # language extensions and the according commment mark
 _LANGS = {'.cc':'*', '.h':'*', '.cu':'*', '.cuh':'*', '.py':'#',
           '.pm':'#', '.scala':'*', '.cc':'*', '.sh':'#', '.cmake':'#',
           '.java':'*', '.sh':'#', '.cpp':'*', '.hpp':'*', '.c':'*',
           '.bat':'rem', '.pl':'#', '.m':'%', '.R':'#', '.mk':'#', '.cfg':'#',
-          '.t':'#', '.ps1':'#', '.jl':'#', '.clj':';;', '.pyx':'#', '.js':'*'}
+          '.t':'#', '.ps1':'#', '.jl':'#', '.clj':';;', '.pyx':'#', '.js':'*',
+          '.md':'<!---'}
 
 # Previous license header, which will be removed
 _OLD_LICENSE = re.compile('.*Copyright.*by Contributors')
@@ -122,8 +143,47 @@ def is_mxnet_root(path: str) -> bool:
     return curpath
 
 
-def _lines_have_license(lines):
-    return any([any([p in l for p in _LICENSE_PATTERNS]) for l in lines])
+def _lines_have_multiple_license(lines):
+    has_apache_license = False
+    has_other_license = False
+    for l in lines:
+        if any(p in l for p in _APACHE_LICENSE_PATTERNS):
+            has_apache_license = True
+        elif any(p in l for p  in _OTHER_LICENSE_PATTERNS):
+            has_other_license = True
+    return (has_apache_license and has_other_license)
+
+
+def _lines_have_apache_license(lines):
+    return any([any([p in l for p in _APACHE_LICENSE_PATTERNS]) for l in lines])
+
+
+def _file_listed_in_top_level_license(fname):
+    with open(TOP_LEVEL_LICENSE_FILE, 'r', encoding="utf-8") as f:
+        lines = f.readlines()
+    module = os.path.split(fname)[0] + '/LICENSE'
+    return any([fname in l or module in l for l in lines])
+
+
+def file_have_valid_license(fname):
+    with open(fname, 'r', encoding="utf-8") as f:
+        lines = f.readlines()
+    if not lines:
+        return True
+    if (_lines_have_apache_license(lines) and (not _lines_have_multiple_license(lines))):
+        return True
+    elif _lines_have_multiple_license(lines):
+        if _file_listed_in_top_level_license(fname):
+            return True
+        else:
+            logging.error("File %s has multiple license", fname)
+            return False
+    else:
+        if _file_listed_in_top_level_license(fname):
+            return True
+        else:
+            logging.error("File %s doesn't have a valid license", fname)
+            return False
 
 
 def _get_license(comment_mark):
@@ -137,6 +197,8 @@ def _get_license(comment_mark):
         body += comment_mark
         if len(l):
             body += ' ' + l
+        if comment_mark == '<!---':
+            body += ' -->'
         body += '\n'
 
     if comment_mark == '*':
@@ -160,13 +222,7 @@ def file_has_license(fname):
     if not should_have_license(fname):
         return True
     try:
-        with open(fname, 'r', encoding="utf-8") as f:
-            lines = f.readlines()
-        if not lines or _lines_have_license(lines):
-            return True
-        else:
-            logging.error("File %s doesn't have a license", fname)
-            return False
+        return file_have_valid_license(fname)
     except UnicodeError:
         return True
     return True
@@ -177,7 +233,7 @@ def file_add_license(fname):
         return
     with open(fname, 'r', encoding="utf-8") as f:
         lines = f.readlines()
-    if _lines_have_license(lines):
+    if _lines_have_apache_license(lines):
         return
     _, ext = os.path.splitext(fname)
     with open(fname, 'w', encoding="utf-8") as f:
@@ -247,7 +303,8 @@ def main():
             files = file_generator(get_mxnet_root())
 
     if action == 'check':
-        if not all(map(file_has_license, files)):
+        logging.info("Start to check %d files", (len(files)))
+        if False in list(map(file_has_license, files)):
             return 1
         else:
             logging.info("All known and whitelisted files have license")