updates to mm after removing vendor directories downstream (#3436)

* updates to mm after removing vendor directories downstream

* more changes to magic-modules for removing vendoring

* fix tests

* lint fixes

* fix more lint errors

* fix more lint errors

* fix more lint errors

* fix the rest of the lint errors

* update generate in ansbile.rb

* update  ansbile.rb

* inspec changes (and some minor terraform stuff)

* updates for ansible

* updates for tf oics and conversion

* made changes for ansible_devel

* updates for linter

.ci/containers/downstream-builder/Dockerfile

@@ -9,8 +9,7 @@ RUN go get github.com/github/hub
 RUN ssh-keyscan github.com >> /known_hosts
 RUN echo "UserKnownHostsFile /known_hosts" >> /etc/ssh/ssh_config
 
-ENV GOFLAGS "-mod=vendor"
-ENV GO111MODULE "off"
+ENV GO111MODULE "on"
 
 # Install Ruby from source.
 RUN apt-get update

.ci/containers/downstream-builder/generate_downstream.sh

@@ -85,7 +85,8 @@ fi
 
 if [ "$REPO" == "terraform" ]; then
     pushd $LOCAL_PATH
-    find . -type f -not -wholename "./.git*" -not -wholename "./.changelog*" -not -wholename "./vendor*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name "GNUmakefile" -not -name "docscheck.sh" -not -name "LICENSE" -not -name "README.md" -not -wholename "./examples*" -not -name "go.mod" -not -name "go.sum" -not -name "staticcheck.conf" -not -name ".go-version" -not -name ".hashibot.hcl" -not -name "tools.go"  -exec git rm {} \;
+    find . -type f -not -wholename "./.git*" -not -wholename "./.changelog*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name "GNUmakefile" -not -name "docscheck.sh" -not -name "LICENSE" -not -name "README.md" -not -wholename "./examples*" -not -name "go.mod" -not -name "go.sum" -not -name "staticcheck.conf" -not -name ".go-version" -not -name ".hashibot.hcl" -not -name "tools.go"  -exec git rm {} \;
+    go mod download
     popd
 fi
 

compile/core.rb

@@ -226,9 +226,9 @@ def compile_string(ctx, source)
       end
     end
 
-    def autogen_notice(lang)
+    def autogen_notice(lang, pwd)
       Thread.current[:autogen] = true
-      comment_block(compile('templates/autogen_notice.erb').split("\n"), lang)
+      comment_block(compile(pwd + '/templates/autogen_notice.erb').split("\n"), lang)
     end
 
     def autogen_exception

products/iam/helpers/ansible/service_account_key_template.erb

@@ -3,12 +3,12 @@
 #
 # Copyright (C) 2017 Google
 # GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
-<%= lines(autogen_notice :python) -%>
+<%= lines(autogen_notice(:python, pwd)) -%>
 
 from __future__ import absolute_import, division, print_function
 __metaclass__ = type
 
-<%= lines(compile('templates/ansible/documentation.erb'), 1) -%>
+<%= lines(compile(pwd + '/templates/ansible/documentation.erb'), 1) -%>
 ################################################################################
 # Imports
 ################################################################################
@@ -48,7 +48,7 @@ import base64
 def main():
     """Main function"""
 
-<%= lines(indent(compile('templates/ansible/module.erb'), 4)) -%>
+<%= lines(indent(compile(pwd + '/templates/ansible/module.erb'), 4)) -%>
 
     if not module.params['scopes']:
         module.params['scopes'] = <%= python_literal(object.__product.scopes) %>

provider/ansible.rb

@@ -94,7 +94,7 @@ def module_name(object)
          object.name.underscore].join('_')
       end
 
-      def build_object_data(object, output_folder, version)
+      def build_object_data(pwd, object, output_folder, version)
         # Method is overridden to add Ansible example objects to the data object.
         data = AnsibleProductFileTemplate.file_for_resource(
           output_folder,
@@ -105,7 +105,7 @@ def build_object_data(object, output_folder, version)
         )
 
         prod_name = data.object.name.underscore
-        path = ["products/#{data.product.api_name}",
+        path = [pwd + "/products/#{data.product.api_name}",
                 "examples/ansible/#{prod_name}.yaml"].join('/')
 
         data.example = get_example(path) if File.file?(path)
@@ -233,19 +233,20 @@ def get_example(cfg_file)
         ex
       end
 
-      def generate_resource(data)
+      def generate_resource(pwd, data)
         target_folder = data.output_folder
         name = module_name(data.object)
         path = File.join(target_folder,
                          "plugins/modules/#{name}.py")
         data.generate(
+          pwd,
           data.object.template || 'templates/ansible/resource.erb',
           path,
           self
         )
       end
 
-      def generate_resource_tests(data)
+      def generate_resource_tests(pwd, data)
         prod_name = data.object.name.underscore
         path = ["products/#{data.product.api_name}",
                 "examples/ansible/#{prod_name}.yaml"].join('/')
@@ -260,6 +261,7 @@ def generate_resource_tests(data)
         path = File.join(target_folder,
                          "tests/integration/targets/#{name}/tasks/main.yml")
         data.generate(
+          pwd,
           'templates/ansible/tests_main.erb',
           path,
           self
@@ -270,6 +272,7 @@ def generate_resource_tests(data)
           path = File.join(target_folder,
                            "tests/integration/targets/#{name}/tasks/#{t.name}.yml")
           data.generate(
+            pwd,
             t.path,
             path,
             self
@@ -280,20 +283,22 @@ def generate_resource_tests(data)
         path = File.join(target_folder,
                          "tests/integration/targets/#{name}/defaults/main.yml")
         data.generate(
+          pwd,
           'templates/ansible/integration_test_variables.erb',
           path,
           self
         )
       end
 
-      def generate_resource_sweepers(data)
+      def generate_resource_sweepers(pwd, data)
         # No generated sweepers for this provider
       end
 
-      def compile_datasource(data)
+      def compile_datasource(pwd, data)
         target_folder = data.output_folder
         name = module_name(data.object)
-        data.generate('templates/ansible/facts.erb',
+        data.generate(pwd,
+                      'templates/ansible/facts.erb',
                       File.join(target_folder,
                                 "plugins/modules/#{name}_info.py"),
                       self)
@@ -344,7 +349,7 @@ def regex_url(url)
     # Generates files on a per-resource basis.
     # All paths are allowed a '%s' where the module name
     # will be added.
-    def generate_resource_files(data)
+    def generate_resource_files(pwd, data)
       return unless @config&.files&.resource
 
       files = @config.files.resource
@@ -358,7 +363,7 @@ def generate_resource_files(data)
         data.version,
         build_env
       )
-      compile_file_list(data.output_folder, files, file_template)
+      compile_file_list(data.output_folder, files, file_template, pwd)
     end
 
     def copy_common_files(output_folder, provider_name = 'ansible')

provider/ansible/example.rb

@@ -229,7 +229,7 @@ class NoVerifier < Verifier
       attr_reader :reason
       def validate() end
 
-      def build_task(_state, _object)
+      def build_task(_state, _object, _pwd)
         ''
       end
     end
@@ -262,9 +262,9 @@ def validate
         true
       end
 
-      def build_task(_state, object)
+      def build_task(_state, object, pwd)
         @parameters = build_parameters(object)
-        compile 'templates/ansible/verifiers/facts.yaml.erb'
+        compile(pwd + '/templates/ansible/verifiers/facts.yaml.erb')
       end
 
       private

provider/ansible_devel.rb

@@ -34,22 +34,24 @@ def module_utils_import_path
         'ansible.module_utils.gcp_utils'
       end
 
-      def generate_resource(data)
+      def generate_resource(pwd, data)
         target_folder = data.output_folder
         name = module_name(data.object)
         path = File.join(target_folder,
                          "lib/ansible/modules/cloud/google/#{name}.py")
         data.generate(
+          pwd,
           data.object.template || 'templates/ansible/resource.erb',
           path,
           self
         )
       end
 
-      def compile_datasource(data)
+      def compile_datasource(pwd, data)
         target_folder = data.output_folder
         name = module_name(data.object)
-        data.generate('templates/ansible/facts.erb',
+        data.generate(pwd,
+                      'templates/ansible/facts.erb',
                       File.join(target_folder,
                                 "lib/ansible/modules/cloud/google/#{name}_info.py"),
                       self)
@@ -64,7 +66,7 @@ def compile_datasource(data)
         File.symlink "#{name}_info.py", deprecated_facts_path
       end
 
-      def generate_resource_tests(data)
+      def generate_resource_tests(pwd, data)
         prod_name = data.object.name.underscore
         path = ["products/#{data.product.api_name}",
                 "examples/ansible/#{prod_name}.yaml"].join('/')
@@ -79,6 +81,7 @@ def generate_resource_tests(data)
         path = File.join(target_folder,
                          "test/integration/targets/#{name}/tasks/main.yml")
         data.generate(
+          pwd,
           'templates/ansible/tests_main.erb',
           path,
           self
@@ -89,14 +92,15 @@ def generate_resource_tests(data)
           path = File.join(target_folder,
                            "test/integration/targets/#{name}/tasks/#{t.name}.yml")
           data.generate(
+            pwd,
             t.path,
             path,
             self
           )
         end
       end
 
-      def generate_resource_sweepers(data) end
+      def generate_resource_sweepers(pwd, data) end
 
       def compile_common_files(_arg1, _arg2, _arg3) end
 
@@ -115,7 +119,7 @@ def copy_common_files(output_folder, provider_name = nil)
         copy_file_list(output_folder, files)
       end
 
-      def generate_resource_files(data)
+      def generate_resource_files(pwd, data)
         return unless @config&.files&.resource
 
         files = @config.files.resource
@@ -132,7 +136,7 @@ def generate_resource_files(data)
           data.version,
           build_env
         )
-        compile_file_list(data.output_folder, files, file_template)
+        compile_file_list(data.output_folder, files, file_template, pwd)
       end
     end
   end

provider/core.rb

@@ -89,10 +89,14 @@ def generate(output_folder, types, product_path, dump_yaml)
       compile_product_files(output_folder) \
         unless @config.files.nil? || @config.files.compile.nil?
 
-      generate_datasources(output_folder, types) \
+      FileUtils.mkpath output_folder unless Dir.exist?(output_folder)
+      pwd = Dir.pwd
+      Dir.chdir output_folder
+      generate_datasources(pwd, output_folder, types) \
         unless @config.datasources.nil?
 
-      generate_operation(output_folder, types)
+      generate_operation(pwd, output_folder, types)
+      Dir.chdir pwd
 
       # Write a file with the final version of the api, after overrides
       # have been applied.
@@ -107,7 +111,7 @@ def generate(output_folder, types, product_path, dump_yaml)
       end
     end
 
-    def generate_operation(output_folder, types); end
+    def generate_operation(pwd, output_folder, types); end
 
     def copy_files(output_folder)
       copy_file_list(output_folder, @config.files.copy)
@@ -181,14 +185,16 @@ def compile_common_files(
       compile_file_list(output_folder, files, file_template)
     end
 
-    def compile_file_list(output_folder, files, file_template)
+    def compile_file_list(output_folder, files, file_template, pwd = Dir.pwd)
+      FileUtils.mkpath output_folder unless Dir.exist?(output_folder)
+      Dir.chdir output_folder
       files.map do |target, source|
         Thread.new do
           Google::LOGGER.debug "Compiling #{source} => #{target}"
-          target_file = File.join(output_folder, target)
-          file_template.generate(source, target_file, self)
+          file_template.generate(pwd, source, target, self)
         end
       end.map(&:join)
+      Dir.chdir pwd
     end
 
     def generate_objects(output_folder, types)
@@ -215,27 +221,34 @@ def generate_objects(output_folder, types)
     end
 
     def generate_object(object, output_folder, version_name)
-      data = build_object_data(object, output_folder, version_name)
+      pwd = Dir.pwd
+      data = build_object_data(pwd, object, output_folder, version_name)
       unless object.exclude_resource
+        FileUtils.mkpath output_folder unless Dir.exist?(output_folder)
+        Dir.chdir output_folder
         Google::LOGGER.debug "Generating #{object.name} resource"
-        generate_resource data.clone
+        generate_resource(pwd, data.clone)
         Google::LOGGER.debug "Generating #{object.name} tests"
-        generate_resource_tests data.clone
-        generate_resource_sweepers data.clone
-        generate_resource_files data.clone
+        generate_resource_tests(pwd, data.clone)
+        generate_resource_sweepers(pwd, data.clone)
+        generate_resource_files(pwd, data.clone)
+        Dir.chdir pwd
       end
 
       # if iam_policy is not defined or excluded, don't generate it
       return if object.iam_policy.nil? || object.iam_policy.exclude
 
+      FileUtils.mkpath output_folder unless Dir.exist?(output_folder)
+      Dir.chdir output_folder
       Google::LOGGER.debug "Generating #{object.name} IAM policy"
-      generate_iam_policy data.clone
+      generate_iam_policy(pwd, data.clone)
+      Dir.chdir pwd
     end
 
     # Generate files at a per-resource basis.
-    def generate_resource_files(data) end
+    def generate_resource_files(pwd, data) end
 
-    def generate_datasources(output_folder, types)
+    def generate_datasources(pwd, output_folder, types)
       # We need to apply overrides for datasources
       @api = Overrides::Runner.build(@api, @config.datasources,
                                      @config.resource_override,
@@ -257,18 +270,18 @@ def generate_datasources(output_folder, types)
             "Excluding #{object.name} datasource per API version"
           )
         else
-          generate_datasource object, output_folder
+          generate_datasource(pwd, object, output_folder)
         end
       end
     end
 
-    def generate_datasource(object, output_folder)
-      data = build_object_data(object, output_folder, @target_version_name)
+    def generate_datasource(pwd, object, output_folder)
+      data = build_object_data(pwd, object, output_folder, @target_version_name)
 
-      compile_datasource data.clone
+      compile_datasource(pwd, data.clone)
     end
 
-    def build_object_data(object, output_folder, version)
+    def build_object_data(_pwd, object, output_folder, version)
       ProductFileTemplate.file_for_resource(output_folder, object, version, @config, build_env)
     end
 
@@ -329,7 +342,7 @@ def update_uri(resource, url_part)
       url_part
     end
 
-    def generate_iam_policy(data) end
+    def generate_iam_policy(pwd, data) end
 
     # TODO(nelsonjr): Review all object interfaces and move to private methods
     # that should not be exposed outside the object hierarchy.