tests: Run rspec tests out of bazel build output

Jenkinsfile

@@ -165,10 +165,16 @@ pipeline {
 
                 withDockerContainer("quay.io/coreos/tectonic-builder:bazel-v0.2") {
                   sh"""#!/bin/bash -ex
-                    bazel build tarball
-                    bazel build tests/smoke
+                    bazel build tarball tests/smoke
                     # bazel test terraform_fmt //installer/frontend:frontend_test --verbose_failures --test_output=errors
+
+                    # Jenkins `stash` does not follow symlinks - thereby temporarily copy the files to the root dir
+                    cp bazel-bin/tectonic.tar.gz .
+                    cp bazel-bin/tests/smoke/linux_amd64_stripped/smoke .
                   """
+
+                  stash name: 'tectonic.tar.gz', includes: 'tectonic.tar.gz'
+                  stash name: 'smoke-tests', includes: 'smoke'
                 }
 
                 withDockerContainer(params.builder_image) {
@@ -190,9 +196,7 @@ pipeline {
                     make test
                     rm -fr frontend/tests_output
                     """
-                    stash name: 'installer-binary', includes: 'installer/bin/linux/installer'
                     stash name: 'node-modules', includes: 'installer/frontend/node_modules/**'
-                    stash name: 'smoke-test-binary', includes: 'bin/smoke'
                   }
                 }
                 withDockerContainer(tectonicSmokeTestEnvImage) {
@@ -468,6 +472,18 @@ def forcefullyCleanWorkspace() {
   }
 }
 
+def unstashCleanRepoTectonicTarGZSmokeTests() {
+  unstash 'clean-repo'
+  unstash 'tectonic.tar.gz'
+  unstash 'smoke-tests'
+  sh """#!/bin/bash -ex
+    # Jenkins `stash` does not follow symlinks - thereby temporarily copy the files to the root dir
+    mkdir -p bazel-bin/tests/smoke/linux_amd64_stripped/
+    cp tectonic.tar.gz bazel-bin/.
+    cp smoke bazel-bin/tests/smoke/linux_amd64_stripped/.
+  """
+}
+
 def runRSpecTest(testFilePath, dockerArgs, credentials) {
   return {
     node('worker && ec2') {
@@ -481,8 +497,7 @@ def runRSpecTest(testFilePath, dockerArgs, credentials) {
                 image: tectonicSmokeTestEnvImage,
                 args: '-u root -v /var/run/docker.sock:/var/run/docker.sock ' + dockerArgs
               ) {
-                unstash 'clean-repo'
-                unstash 'smoke-test-binary'
+                unstashCleanRepoTectonicTarGZSmokeTests()
                 sh """#!/bin/bash -ex
                   mkdir -p templogfiles && chmod 777 templogfiles
                   cd tests/rspec
@@ -500,7 +515,7 @@ def runRSpecTest(testFilePath, dockerArgs, credentials) {
       } finally {
         reportStatusToGithub((err == null) ? 'success' : 'failure', testFilePath, originalCommitId)
         step([$class: "TapPublisher", testResults: "templogfiles/*", outputTapToConsole: true, planRequired: false])
-        archiveArtifacts allowEmptyArchive: true, artifacts: 'build/**/logs/**'
+        archiveArtifacts allowEmptyArchive: true, artifacts: 'bazel-bin/tectonic/build/**/logs/**'
         withDockerContainer(params.builder_image) {
          withCredentials(credsUI) {
           script {
@@ -530,8 +545,7 @@ def runRSpecTestBareMetal(testFilePath, credentials) {
       try {
         timeout(time: 5, unit: 'HOURS') {
           ansiColor('xterm') {
-            unstash 'clean-repo'
-            unstash 'smoke-test-binary'
+            unstashCleanRepoTectonicTarGZSmokeTests()
             withCredentials(credentials + quayCreds) {
               sh """#!/bin/bash -ex
               cd tests/rspec
@@ -552,7 +566,7 @@ def runRSpecTestBareMetal(testFilePath, credentials) {
       } finally {
         reportStatusToGithub((err == null) ? 'success' : 'failure', testFilePath, originalCommitId)
         step([$class: "TapPublisher", testResults: "../../templogfiles/*", outputTapToConsole: true, planRequired: false])
-        archiveArtifacts allowEmptyArchive: true, artifacts: 'build/**/logs/**'
+        archiveArtifacts allowEmptyArchive: true, artifacts: 'bazel-bin/tectonic/build/**/logs/**'
         withCredentials(credsUI) {
           script {
             try {

tests/rspec/lib/aws_vpc.rb

@@ -43,7 +43,7 @@ def export_tfvars
   end
 
   def create
-    Dir.chdir('../../contrib/internal-cluster') do
+    Dir.chdir(File.join(ENV['RSPEC_PATH'], '../../contrib/internal-cluster')) do
       succeeded = system(env_variables, 'terraform init')
       raise 'could not init Terraform to create VPC' unless succeeded
       succeeded = system(env_variables, 'terraform apply -auto-approve')
@@ -87,7 +87,7 @@ def destroy
   end
 
   def terraform_destroy
-    Dir.chdir('../../contrib/internal-cluster') do
+    Dir.chdir(File.join(ENV['RSPEC_PATH'], '../../contrib/internal-cluster')) do
       3.times do
         return if system(env_variables, 'terraform destroy -force')
       end

tests/rspec/lib/azure_vpn.rb

@@ -9,7 +9,7 @@
 require 'net/ssh'
 require 'azure_support'
 
-AZURE_VPN_TEMPLATES = '../smoke/azure/fixtures/private-cluster/*.tf'
+AZURE_VPN_TEMPLATES = File.join(ENV['RSPEC_PATH'], '../smoke/azure/fixtures/private-cluster/*.tf')
 MAX_RETRIES = 3
 TIMEOUT_RETRIES = 30
 TIMEOUT_RETRY_DELAY = 3

tests/rspec/lib/cluster.rb

@@ -30,7 +30,7 @@ def initialize(tfvars_file)
     @tectonic_admin_password = ENV['TF_VAR_tectonic_admin_password'] || PasswordGenerator.generate_password
     save_console_creds(@name, @tectonic_admin_email, @tectonic_admin_password)
 
-    @build_path = File.join(File.realpath('../../'), "build/#{@name}")
+    @build_path = File.join(File.dirname(ENV['RELEASE_TARBALL_PATH']), "tectonic/build/#{@name}")
     @manifest_path = File.join(@build_path, 'generated')
     @kubeconfig = File.join(manifest_path, 'auth/kubeconfig')
     @tfstate_file = TFStateFile.new(@build_path)
@@ -177,7 +177,7 @@ def apply
         env['TF_APPLY_OPTIONS'] = '-no-color'
         env['TF_INIT_OPTIONS'] = '-no-color'
 
-        return run_command(env, 'apply')
+        return run_command(env, 'apply', '-auto-approve')
       end
     end
   rescue Timeout::Error
@@ -191,7 +191,7 @@ def destroy
         env = env_variables
         env['TF_DESTROY_OPTIONS'] = '-no-color'
         env['TF_INIT_OPTIONS'] = '-no-color'
-        return run_command(env, 'destroy')
+        return run_command(env, 'destroy', '-force')
       end
     end
 
@@ -213,9 +213,10 @@ def terraform_init
     raise 'Terraform init failed'
   end
 
-  def run_command(env, cmd)
-    Open3.popen3(env, "bash -co pipefail 'make -C ../.. #{cmd} |
-                    tee ../../build/#{@name}/terraform-#{cmd}.log'") do |_stdin, stdout, stderr, wait_thr|
+  def run_command(env, cmd, flags = '')
+    command = "terraform #{cmd} #{flags} -var-file=terraform.tfvars ../../platforms/#{env_variables['PLATFORM']} |
+      tee terraform-#{cmd}.log"
+    Open3.popen3(env, "bash -coxe pipefail '#{command}'") do |_stdin, stdout, stderr, wait_thr|
       while (line = stdout.gets)
         puts line
       end

tests/rspec/lib/cluster_support.rb

@@ -41,7 +41,6 @@ def print_service_logs(destination_ip, service, cluster_name, via_host_ip = nil)
     output += "Standard output: \n#{stdout}"
     output += "\nStandard error: \n#{stderr}"
     output += "\nEnd of journal of #{service} service on #{destination_ip}"
-    puts output
 
     save_to_file(cluster_name, 'systemd', destination_ip, service, output)
   rescue => e

tests/rspec/lib/govcloud_vpc.rb

@@ -51,7 +51,7 @@ def export_tfvars
   end
 
   def create
-    Dir.chdir('../../contrib/govcloud') do
+    Dir.chdir(File.join(ENV['RSPEC_PATH'], '../../contrib/govcloud')) do
       succeeded = system(env_variables, 'terraform init')
       raise 'could not init Terraform to create VPC' unless succeeded
       succeeded = system(env_variables, 'terraform apply -auto-approve')
@@ -88,15 +88,15 @@ def parse_subnets(tf_out)
 
   def destroy
     @vpn_connection.stop
-  rescue
-    raise 'could not disconnect from vpn'
+  rescue => e
+    raise 'could not disconnect from vpn:' + e
   ensure
     terraform_destroy
     recover_etc_resolv
   end
 
   def terraform_destroy
-    Dir.chdir('../../contrib/govcloud') do
+    Dir.chdir(File.join(ENV['RSPEC_PATH'], '../../contrib/govcloud')) do
       3.times do
         return if system(env_variables, 'terraform destroy -force')
       end

tests/rspec/lib/metal_support.rb

@@ -37,9 +37,8 @@ def self.install_base_software
 
   def self.setup_bare(varfile)
     # Copy the certificates to matchbox folder
-    tectonic_folder = File.expand_path('../', Dir.pwd)
     root = root_path
-    certs = Dir["#{tectonic_folder}/smoke/bare-metal/fake-creds/{ca.crt,server.crt,server.key}"]
+    certs = Dir[File.join(ENV['RSPEC_PATH'], '../smoke/bare-metal/fake-creds/{ca.crt,server.crt,server.key}')]
     certs.each do |cert|
       filename_dest = cert.split('/')[-1]
       dest_folder = "#{root}/matchbox/examples/etc/matchbox/#{filename_dest}"
@@ -55,12 +54,12 @@ def self.setup_bare(varfile)
 
     # Setting up the metal0 bridge
     execute_command('sudo mkdir -p /etc/rkt/net.d')
-    execute_command("sudo cp #{root}/tests/rspec/utils/20-metal.conf /etc/rkt/net.d/")
+    execute_command('sudo cp ' + File.join(ENV['RSPEC_PATH'], 'utils/20-metal.conf') + ' /etc/rkt/net.d/')
     execute_command('cat /etc/rkt/net.d/20-metal.conf')
 
     # Setting up auth to download images from quay.io
     execute_command('sudo mkdir -p /etc/rkt/auth.d')
-    rkt_auth_file = File.read("#{root}/tests/rspec/utils/rkt-auth.json")
+    rkt_auth_file = File.read(File.join(ENV['RSPEC_PATH'], 'utils/rkt-auth.json'))
     data_hash = JSON.parse(rkt_auth_file)
     data_hash['credentials']['user'] = ENV['QUAY_ROBOT_USERNAME']
     data_hash['credentials']['password'] = ENV['QUAY_ROBOT_SECRET']
@@ -175,14 +174,14 @@ def self.print_service_logs(service)
   end
 
   def self.save_to_file(service_name, output)
-    logs_path = "#{root_path}build/#{ENV['CLUSTER']}/logs/systemd"
+    logs_path = "#{root_path}/build/#{ENV['CLUSTER']}/logs/systemd"
     save_file = File.open("#{logs_path}/#{service_name}.log", 'w+')
     save_file << output
     save_file.close
   end
 
   def self.root_path
-    File.expand_path('../../', Dir.pwd)
+    File.join(File.dirname(ENV['RELEASE_TARBALL_PATH']), 'tectonic')
   end
 
   def self.env_variables_setup

tests/rspec/lib/shared_examples/build_folder_setup.rb

@@ -5,16 +5,18 @@
 
 RSpec.shared_examples 'withBuildFolderSetup' do |tf_vars_path|
   before(:all) do
+    Dir.chdir(File.join(File.dirname(ENV['RELEASE_TARBALL_PATH']), 'tectonic'))
     temp_tfvars_file = TFVarsFile.new(tf_vars_path)
     @name = ENV['CLUSTER'] || NameGenerator.generate(temp_tfvars_file.prefix)
     ENV['CLUSTER'] = @name
-    file_path = "../../build/#{@name}"
+    file_path = "build/#{@name}"
     FileUtils.mkdir_p file_path
+    Dir.chdir(file_path)
 
     FileUtils.cp(
       tf_vars_path,
-      Dir.pwd + "/#{file_path}/terraform.tfvars"
+      'terraform.tfvars'
     )
-    @tfvars_file = TFVarsFile.new("#{file_path}/terraform.tfvars")
+    @tfvars_file = TFVarsFile.new(File.expand_path('terraform.tfvars'))
   end
 end

tests/rspec/lib/shared_examples/k8s.rb

@@ -36,11 +36,8 @@
                  ClusterFactory.from_variable(exist_plat, exist_tf)
                end
 
-    if exist_plat.nil? && exist_tf.nil?
-      @cluster.start
-    else
-      @cluster.init
-    end
+    @cluster.init
+    @cluster.start if exist_plat.nil? && exist_tf.nil?
   end
 
   # after(:all) hooks that are defined first are executed last

tests/rspec/lib/shared_examples/tls_setup.rb

@@ -5,10 +5,10 @@
 
 RSpec.shared_examples 'withTLSSetup' do |domain|
   before(:all) do
-    test_folder = File.expand_path('..', Dir.pwd)
+    test_folder = File.join(ENV['RSPEC_PATH'], '..')
     generate_tls("#{test_folder}/smoke/user_provided_tls/certs/", @name, domain, @tfvars_file.etcd_count)
 
-    root_folder = File.expand_path('../..', Dir.pwd)
+    root_folder = File.join(ENV['RSPEC_PATH'], '../..')
     custom_tls_tf = "#{test_folder}/smoke/user_provided_tls/tls.tf"
     dest_folder = "#{root_folder}/platforms/#{@tfvars_file.platform}"
     original_tls_tf = "#{dest_folder}/tls.tf"

tests/rspec/lib/smoke_test.rb

@@ -5,27 +5,18 @@
 # SmokeTest contains helper functions to operate the smoke tests written in
 # golang
 module SmokeTest
-  def self.build
-    succeeded = system('make -C ../.. bin/smoke')
-    raise 'Could not build smoke test binary' unless succeeded
-  end
-
   def self.run(cluster)
     ::Timeout.timeout(30 * 60) do # 30 minutes
-      build unless compiled?
-
       succeeded = system(
         env_variables(cluster),
-        './../../bin/smoke -test.v -test.parallel=1 --cluster'
+        File.join(
+          File.dirname(ENV['RELEASE_TARBALL_PATH']), 'tests/smoke/linux_amd64_stripped/smoke'
+        ) + ' -test.v -test.parallel=1 --cluster'
       )
       raise 'SmokeTests failed' unless succeeded
     end
   end
 
-  def self.compiled?
-    File.file?('../../bin/smoke')
-  end
-
   def self.env_variables(cluster)
     {
       'SMOKE_KUBECONFIG' => cluster.kubeconfig,

tests/rspec/spec/aws/basic_spec.rb

@@ -3,13 +3,14 @@
 require 'shared_examples/k8s'
 
 RSpec.describe 'aws-standard' do
-  include_examples('withBuildFolderSetup', '../smoke/aws/vars/aws.tfvars.json')
+  include_examples('withBuildFolderSetup', File.join(ENV['RSPEC_PATH'], '../smoke/aws/vars/aws.tfvars.json'))
 
   before(:all) do
-    # doing this because the example aws.tfvars.json is being used by the UI tests
-    # and it generate a fix cluster name. Here we just change the cluster/dns name
-    # all the other content stay the same as generated by the UI
-    # If we use the same name generated by the UI we can run just one test at the time.
+    # doing this because the example aws.tfvars.json is being used by the UI
+    # tests and it generates a fix cluster name. Here we just change the
+    # cluster/dns name all the other content stays the same as generated by the
+    # UI If we use the same name generated by the UI we can run just one test at
+    # the time.
     @tfvars_file.change_cluster_name(@name)
     @tfvars_file.change_dns_name(@name)
   end

tests/rspec/spec/aws/ca_spec.rb

@@ -5,6 +5,6 @@
 RSpec.describe 'aws-custom-ca' do
   include_examples(
     'withRunningCluster',
-    '../smoke/aws/vars/aws-ca.tfvars.json'
+    File.join(ENV['RSPEC_PATH'], '../smoke/aws/vars/aws-ca.tfvars.json')
   )
 end

tests/rspec/spec/aws/custom_tls_spec.rb

@@ -5,7 +5,7 @@
 DOMAIN = 'tectonic-ci.de'
 
 RSpec.describe 'aws-custom-tls' do
-  include_examples('withBuildFolderSetup', '../smoke/aws/vars/aws.tfvars.json')
+  include_examples('withBuildFolderSetup', File.join(ENV['RSPEC_PATH'], '../smoke/aws/vars/aws.tfvars.json'))
   include_examples('withTLSSetup', DOMAIN)
 
   before(:all) do

tests/rspec/spec/aws/exp_spec.rb

@@ -5,6 +5,6 @@
 RSpec.describe 'aws-exp' do
   include_examples(
     'withRunningCluster',
-    '../smoke/aws/vars/aws-exp.tfvars.json'
+    File.join(ENV['RSPEC_PATH'], '../smoke/aws/vars/aws-exp.tfvars.json')
   )
 end

tests/rspec/spec/aws/network_flannel_spec.rb

@@ -5,6 +5,6 @@
 RSpec.describe 'aws-network-flannel' do
   include_examples(
     'withRunningCluster',
-    '../smoke/aws/vars/aws-net-flannel.tfvars.json'
+    File.join(ENV['RSPEC_PATH'], '../smoke/aws/vars/aws-net-flannel.tfvars.json')
   )
 end

tests/rspec/spec/aws/vpc_internal_spec.rb

@@ -7,7 +7,10 @@
 require 'aws_iam'
 
 RSpec.describe 'aws-vpc' do
-  include_examples('withBuildFolderSetup', '../smoke/aws/vars/aws-vpc-internal.tfvars.json')
+  include_examples(
+    'withBuildFolderSetup',
+    File.join(ENV['RSPEC_PATH'], '../smoke/aws/vars/aws-vpc-internal.tfvars.json')
+  )
 
   before(:all) do
     export_random_region_if_not_defined

tests/rspec/spec/azure/basic_spec.rb

@@ -3,5 +3,5 @@
 require 'shared_examples/k8s'
 
 RSpec.describe 'azure-basic' do
-  include_examples('withRunningCluster', '../smoke/azure/vars/basic.tfvars')
+  include_examples('withRunningCluster', File.join(ENV['RSPEC_PATH'], '../smoke/azure/vars/basic.tfvars'))
 end

tests/rspec/spec/azure/custom_tls_spec.rb

@@ -9,6 +9,11 @@
     ENV['TF_VAR_tectonic_azure_location'] = azure_region
     @domain = "#{azure_region}.cloudapp.azure.com"
 
-    include_examples('withRunningClusterWithCustomTLS', '../smoke/azure/vars/basic.tfvars', @domain, false)
+    include_examples(
+      'withRunningClusterWithCustomTLS',
+      File.join(ENV['RSPEC_PATH'], '../smoke/azure/vars/basic.tfvars'),
+      @domain,
+      false
+    )
   end
 end