Refactor yaml check and submit task

Signed-off-by: EdmondFrank <[email protected]>

Gemfile

@@ -21,6 +21,10 @@ gem 'jbuilder'
 # Use Redis adapter to run Action Cable in production
 gem 'redis'
 
+# User RabbitMQ
+gem 'bunny'
+gem 'sneakers'
+
 # Use OpenSearch
 gem 'search_flip'
 gem 'opensearch-ruby'

Gemfile.lock

@@ -68,6 +68,7 @@ GEM
       tzinfo (~> 2.0)
     addressable (2.8.0)
       public_suffix (>= 2.0.2, < 5.0)
+    amq-protocol (2.3.2)
     annotate (3.2.0)
       activerecord (>= 3.2, < 8.0)
       rake (>= 10.4, < 14.0)
@@ -81,6 +82,9 @@ GEM
     bundler-audit (0.9.1)
       bundler (>= 1.2.0, < 3)
       thor (~> 1.0)
+    bunny (2.19.0)
+      amq-protocol (~> 2.3, >= 2.3.1)
+      sorted_set (~> 1, >= 1.0.2)
     capybara (3.37.1)
       addressable
       matrix
@@ -306,6 +310,7 @@ GEM
       activerecord (>= 6.1.5)
       activesupport (>= 6.1.5)
       i18n
+    rbtree (0.4.5)
     redis (4.6.0)
     regexp_parser (2.4.0)
     reline (0.3.1)
@@ -371,17 +376,30 @@ GEM
       childprocess (>= 0.5, < 5.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2)
+    serverengine (2.0.7)
+      sigdump (~> 0.2.2)
+    set (1.0.3)
     sexp_processor (4.16.1)
     sidekiq (6.4.2)
       connection_pool (>= 2.2.2)
       rack (~> 2.0)
       redis (>= 4.2.0)
+    sigdump (0.2.4)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
     simplecov-html (0.12.3)
     simplecov_json_formatter (0.1.4)
+    sneakers (2.11.0)
+      bunny (~> 2.12)
+      concurrent-ruby (~> 1.0)
+      rake
+      serverengine (~> 2.0.5)
+      thor
+    sorted_set (1.0.3)
+      rbtree
+      set (~> 1.0)
     sprockets (4.0.3)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -434,6 +452,7 @@ DEPENDENCIES
   bootsnap
   brakeman
   bundler-audit
+  bunny
   capybara
   crono
   database_cleaner-active_record
@@ -475,6 +494,7 @@ DEPENDENCIES
   selenium-webdriver
   sidekiq
   simplecov
+  sneakers
   sprockets-rails
   tzinfo-data
   web-console

Rakefile

@@ -4,3 +4,5 @@
 require_relative "config/application"
 
 Rails.application.load_tasks
+
+require 'sneakers/tasks'

app/controllers/application_controller.rb

@@ -17,29 +17,29 @@ def workflow
 
     action = payload['action']
     action_desc = payload['action_desc']
-    @pr_number = payload['iid'] || payload['number']
-    @merged_at = payload['pull_request']&.[]('merged_at')
-    result =
+    merged_at = payload['pull_request']&.[]('merged_at')
+
+    queue =
       case action
       when 'open', 'reopen', 'opened', 'synchronize'
-        check_yaml(payload)
+        'yaml_check_v1'
       when 'update'
         if action_desc == 'source_branch_changed'
-          check_yaml(payload)
+          'yaml_check_v1'
         end
       when 'merge'
-        submit_task(payload)
+        'submit_task_v1'
       when 'closed'
-        if @merged_at.present?
-          submit_task(payload)
+        if merged_at.present?
+          'submit_task_v1'
         end
       end
 
-    if result.present? && result.is_a?(Hash)
-      notify_on_pr(@pr_number, quote_mark(YAML.dump(result)))
+    if queue
+      RabbitMQ.publish(queue, { user_agent: user_agent, payload: payload })
     end
 
-    render json: result
+    render json: { status: true, message: 'Your submission is being processed' }
   end
 
   def hook
@@ -67,106 +67,18 @@ def panel
 
   protected
 
-  def check_yaml(payload)
-    diff_url = payload&.[]('pull_request')&.[]('diff_url')
-    @branch = payload&.[]('pull_request')&.[]('head')&.[]('ref')
-    if diff_url.present?
-      result = []
-      req = { method: :get, url: diff_url }
-      req.merge!(proxy: PROXY) unless extract_domain(diff_url).start_with?('gitee')
-      diff = RestClient::Request.new(req).execute.body
-      patches = GitDiffParser.parse(diff)
-      patches.each do |patch|
-        if patch.file.start_with?(SINGLE_DIR)
-          result << analyze_yaml_file(patch.file)
-        elsif patch.file.start_with?(ORG_DIR)
-          result << analyze_org_yaml_file(patch.file)
-        else
-          result << { status: false, message: "invaild configure yaml path: #{patch.file}" }
-        end
-      end
-      { status: true, message: 'ok', result: result }
-    else
-      { status: false, message: 'invalid diff url' }
-    end
-  rescue => ex
-    { status: false, message: ex.message }
-  end
-
-  def submit_task(payload)
-    diff_url = payload&.[]('pull_request')&.[]('diff_url')
-    @branch = payload&.[]('pull_request')&.[]('base')&.[]('ref')
-    if diff_url.present?
-      result = []
-      req = { method: :get, url: diff_url }
-      req.merge!(proxy: PROXY) unless extract_domain(diff_url).start_with?('gitee')
-      diff = RestClient::Request.new(req).execute.body
-      patches = GitDiffParser.parse(diff)
-      patches.each do |patch|
-        if patch.file.start_with?(SINGLE_DIR)
-          result << analyze_yaml_file(patch.file, only_validate: false)
-        elsif patch.file.start_with?(ORG_DIR)
-          result << analyze_org_yaml_file(patch.file, only_validate: false)
-        else
-          result << { status: false, message: "invaild configure yaml path: #{patch.file}" }
-        end
-      end
-      { status: true, message: 'ok', result: result }
-    else
-      { status: false, message: 'invalid diff url' }
-    end
-  rescue => ex
-    { status: false, message: ex.message }
-  end
-
-  def analyze_yaml_file(path, only_validate: true)
-    yaml_url = generate_yml_url(path)
-    req = { method: :get, url: yaml_url }
-    req.merge!(proxy: PROXY) unless extract_domain(yaml_url).start_with?('gitee')
-    yaml = YAML.load(RestClient::Request.new(req).execute.body)
-    AnalyzeServer.new(
-      {
-        repo_url: yaml['resource_types']['repo_urls'],
-        raw: true,
-        enrich: true,
-        activity: true,
-        community: true,
-        codequality: true,
-        group_activity: true,
-        callback: {
-          hook_url: "#{HOST}/api/hook",
-          params: { pr_number: @pr_number }
-        }
-      }
-    ).execute(only_validate: only_validate).merge(path: path)
-  rescue => ex
-    { status: false, path: path, message: ex.message }
+  def render_json(status_code = 200, status: status_code, data: nil, message: nil)
+    render json: { status: status, data: data, message: message }, status: status_code
   end
 
-  def analyze_org_yaml_file(path, only_validate: true)
-    yaml_url = generate_yml_url(path)
-    AnalyzeGroupServer.new(
-      {
-        yaml_url: yaml_url,
-        raw: true,
-        enrich: true,
-        activity: true,
-        community: true,
-        codequality: true,
-        group_activity: true,
-        callback: {
-          hook_url: "#{HOST}/api/hook",
-          params: { pr_number: @pr_number }
-        }
-      }
-    ).execute(only_validate: only_validate).merge(path: path)
+  def notify_on_pr(pr_number, message, domain: nil)
+    notify_method =
+      gitee_agent?(user_agent) || domain == 'gitee' ?
+        :gitee_notify_on_pr :
+        :github_notify_on_pr
+    self.send(notify_method, owner(user_agent), repo(user_agent), pr_number, message)
   rescue => ex
-    { status: false, path: path, message: ex.message }
-  end
-
-
-  def render_json(status_code = 200, status: status_code, data: nil, message: nil)
-    render json: { status: status, data: data, message: message }, status: status_code
+    Rails.logger.error("Failed to notify on pr #{pr_number}, #{ex.message}")
   end
 
   private
@@ -175,45 +87,7 @@ def user_agent
     @user_agent ||= request.user_agent
   end
 
-  def gitee_agent?
-    user_agent == 'git-oschina-hook'
-  end
-
-  def quote_mark(message)
-    <<~HEREDOC
-    ```
-    #{message}
-    ```
-    HEREDOC
-  end
-
-  def generate_yml_url(path)
-    if gitee_agent?
-      "#{GITEE_REPO}/raw/#{@branch}/#{path}"
-    else
-      "#{GITHUB_REPO.sub('github.com', 'raw.githubusercontent.com')}/#{@branch}/#{path}"
-    end
-  end
-
-  def owner
-    (gitee_agent? ? GITEE_REPO : GITHUB_REPO).split('/')[-2]
-  end
-
-  def repo
-    (gitee_agent? ? GITEE_REPO : GITHUB_REPO).split('/')[-1]
-  end
-
-  def notify_on_pr(pr_number, message, domain: nil)
-    if gitee_agent? || domain == 'gitee'
-      gitee_notify_on_pr(owner, repo, pr_number, message)
-    else
-      github_notify_on_pr(owner, repo, pr_number, message)
-    end
-  rescue => ex
-    logger.error("Failed to notify on pr #{pr_number}, #{ex.message}")
-  end
-
   def auth_validate
-    gitee_agent? ? gitee_webhook_verify : github_webhook_verify
+    gitee_agent?(user_agent) ? gitee_webhook_verify : github_webhook_verify
   end
 end

app/controllers/concerns/common.rb

@@ -7,10 +7,83 @@ module Common
   CELERY_SERVER = ENV.fetch('CELERY_SERVER') { 'http://localhost:8000' }
   SUPPORT_DOMAINS = ['gitee.com', 'github.com', 'raw.githubusercontent.com']
   SUPPORT_DOMAIN_NAMES = ['gitee', 'github']
+  GITEE_REPO = ENV.fetch('GITEE_WORKFLOW_REPO')
+  GITHUB_REPO = ENV.fetch('GITHUB_WORKFLOW_REPO')
 
   Faraday.ignore_env_proxy = true
 
   def extract_domain(url)
     Addressable::URI.parse(url)&.normalized_host
   end
+
+  def quote_mark(message)
+    <<~HEREDOC
+    ```
+    #{message}
+    ```
+    HEREDOC
+  end
+
+  def gitee_agent?(agent)
+    agent == 'git-oschina-hook'
+  end
+
+  def owner(agent)
+    (gitee_agent?(agent) ? GITEE_REPO : GITHUB_REPO).split('/')[-2]
+  end
+
+  def repo(agent)
+    (gitee_agent?(agent) ? GITEE_REPO : GITHUB_REPO).split('/')[-1]
+  end
+
+  def generate_yml_url(agent, branch, path)
+    if gitee_agent?(agent)
+      "#{GITEE_REPO}/raw/#{branch}/#{path}"
+    else
+      "#{GITHUB_REPO.sub('github.com', 'raw.githubusercontent.com')}/#{branch}/#{path}"
+    end
+  end
+
+  def each_patch_with_action(diff_url, &action)
+    req = { method: :get, url: diff_url }
+    req.merge!(proxy: PROXY) unless extract_domain(diff_url).start_with?('gitee')
+    diff = RestClient::Request.new(req).execute.body
+    patches = GitDiffParser.parse(diff)
+    patches.each do |patch|
+      action.call(patch)
+    end
+  end
+
+  def analyze_or_submit_yaml_file(analyzer, user_agent, branch, path, extra={})
+
+    yaml_url = generate_yml_url(user_agent, branch, path)
+    is_org = extra[:is_org] || false
+    pr_number = extra[:pr_number]
+    only_validate = extra[:only_validate]
+
+    base_config = {
+      raw: true,
+      enrich: true,
+      activity: true,
+      community: true,
+      codequality: true,
+      group_activity: true,
+      callback: {
+        hook_url: "#{HOST}/api/hook",
+        params: { pr_number: pr_number }
+      }
+    }
+    params =
+      if is_org
+        base_config.merge(yaml_url: yaml_url)
+      else
+        req = { method: :get, url: yaml_url }
+        req.merge!(proxy: PROXY) unless extract_domain(yaml_url).start_with?('gitee')
+        yaml = YAML.load(RestClient::Request.new(req).execute.body)
+        base_config.merge(repo_url: yaml['resource_types']['repo_urls'])
+      end
+    analyzer.new(params).execute(only_validate: only_validate).merge(path: path)
+  rescue => ex
+    { status: false, message: ex.message, url: yaml_url }
+  end
 end

app/controllers/concerns/gitee_application.rb

@@ -3,7 +3,6 @@ module GiteeApplication
   include Common
 
   GITEE_TOKEN = ENV.fetch('GITEE_API_TOKEN')
-  GITEE_REPO = ENV.fetch('GITEE_WORKFLOW_REPO')
   GITEE_API_ENDPOINT = 'https://gitee.com/api/v5'
 
   def gitee_notify_on_pr(owner, repo, pr_number, message)
@@ -69,7 +68,7 @@ def gitee_create_pull(title, content, head, base: 'main')
           body: content,
           head: head,
           base: base,
-          prune_source_branch: true,
+          prune_source_branch: false,
           access_token: GITEE_TOKEN
         }.to_json,
         { 'Content-Type' => 'application/json'}