feat: 🎸 Google Natural Language API を実装 (#102)

.env.sample

@@ -3,6 +3,11 @@ POSTGRES_PORT_DEVELOPMENT=5432
 POSTGRES_USERNAME_DEVELOPMENT=username
 POSTGRES_PASSWORD_DEVELOPMENT=password
 
+POSTGRES_HOST_TEST=
+POSTGRES_PORT_TEST=
+POSTGRES_USERNAME_TEST=
+POSTGRES_PASSWORD_TEST=
+
 TWEET_STORAGE_DATABASE=
 TWEET_STORAGE_HOST=
 TWEET_STORAGE_PORT=
@@ -22,3 +27,5 @@ TWITTER_CONSUMER_KEY=
 TWITTER_CONSUMER_SECRET=
 TWITTER_ACCESS_TOKEN=
 TWITTER_ACCESS_SECRET=
+
+LANGUAGE_CREDENTIALS=google_natural_language_api_credentials.json

Gemfile

@@ -9,6 +9,7 @@ gem 'bootsnap', require: false
 gem 'bugsnag'
 gem 'dotenv-rails'
 gem 'google-apis-sheets_v4'
+gem 'google-cloud-language'
 gem 'paper_trail'
 gem 'pg'
 gem 'puma'
@@ -19,12 +20,12 @@ gem 'twitter'
 group :development, :test do
   gem 'byebug'
   gem 'factory_bot_rails'
+  gem 'pry-rails'
   gem 'rspec-rails'
 end
 
 group :development do
   gem 'listen'
-  gem 'pry-rails'
   gem 'rubocop-rails'
   gem 'spring'
 end

Gemfile.lock

@@ -105,6 +105,13 @@ GEM
     ffi-compiler (1.0.1)
       ffi (>= 1.0.0)
       rake
+    gapic-common (0.4.1)
+      faraday (~> 1.3)
+      google-protobuf (~> 3.15, >= 3.15.2)
+      googleapis-common-protos (>= 1.3.11, < 2.0)
+      googleapis-common-protos-types (>= 1.0.6, < 2.0)
+      googleauth (~> 0.15, >= 0.15.1)
+      grpc (~> 1.36)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
     google-apis-core (0.3.0)
@@ -119,13 +126,39 @@ GEM
       webrick
     google-apis-sheets_v4 (0.6.0)
       google-apis-core (~> 0.1)
+    google-cloud-core (1.6.0)
+      google-cloud-env (~> 1.0)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.5.0)
+      faraday (>= 0.17.3, < 2.0)
+    google-cloud-errors (1.1.0)
+    google-cloud-language (1.3.0)
+      google-cloud-core (~> 1.5)
+      google-cloud-language-v1 (~> 0.1)
+      google-cloud-language-v1beta2 (~> 0.1)
+    google-cloud-language-v1 (0.4.0)
+      gapic-common (~> 0.3)
+      google-cloud-errors (~> 1.0)
+    google-cloud-language-v1beta2 (0.4.0)
+      gapic-common (~> 0.3)
+      google-cloud-errors (~> 1.0)
+    google-protobuf (3.17.2-x86_64-linux)
+    googleapis-common-protos (1.3.11)
+      google-protobuf (~> 3.14)
+      googleapis-common-protos-types (>= 1.0.6, < 2.0)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.0.6)
+      google-protobuf (~> 3.14)
     googleauth (0.16.2)
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.14)
+    grpc (1.38.0-x86_64-linux)
+      google-protobuf (~> 3.15)
+      googleapis-common-protos-types (~> 1.0)
     http (4.4.1)
       addressable (~> 2.3)
       http-cookie (~> 1.0)
@@ -314,6 +347,7 @@ DEPENDENCIES
   dotenv-rails
   factory_bot_rails
   google-apis-sheets_v4
+  google-cloud-language
   listen
   paper_trail
   pg

app/lib/google_natural_language_api/pickup_character_names.rb

@@ -0,0 +1,39 @@
+# https://googleapis.dev/ruby/google-cloud-language/latest/file.MIGRATING.html
+# https://googleapis.dev/ruby/google-cloud-language-v1/latest/Google/Cloud/Language/V1/AnalyzeSyntaxResponse.html
+# https://cloud.google.com/natural-language/#natural-language-api-demo
+# GoogleNaturalLanguageApi::PickupCharacterNames.new.foobar
+
+require "google/cloud/language"
+
+module GoogleNaturalLanguageApi
+  class PickupCharacterNames
+    attr_reader :client
+
+    def initialize
+      @language = Google::Cloud::Language.language_service
+      @client = Google::Cloud::Language.language_service
+    end
+
+    def create_analyze_syntax_object(tweet)
+      response = analyze_tweet_syntax_by_api(tweet)
+
+      ActiveRecord::Base.transaction do
+        analyze_syntax = AnalyzeSyntax.new(
+          language: response.language,
+          sentences: response.sentences.map(&:to_json),
+          tokens: response.tokens.map(&:to_json),
+          tweet_id: tweet.id
+        )
+
+        analyze_syntax.save!
+      end
+    end
+
+    def analyze_tweet_syntax_by_api(tweet)
+      content = tweet.full_text
+      document = { type: :PLAIN_TEXT, content: content }
+
+      @language.analyze_syntax(document: document)
+    end
+  end
+end

app/models/analyze_syntax.rb

@@ -0,0 +1,27 @@
+class AnalyzeSyntax < ApplicationRecord
+  belongs_to :tweet
+
+  def convert_analyze_syntax_response_sentence_objects
+    hashed_sentences.map do |hashed_sentence|
+      hashed_sentence.merge!(analyze_syntax_id: id)
+
+      AnalyzeSyntaxResponse::Sentence.new(hashed_sentence)
+    end
+  end
+
+  def convert_analyze_syntax_response_token_objects
+    hashed_tokens.map do |hashed_token|
+      hashed_token.merge!(analyze_syntax_id: id)
+
+      AnalyzeSyntaxResponse::Token.new(hashed_token)
+    end
+  end
+
+  def hashed_tokens
+    tokens.map { |token| JSON.parse(token) }
+  end
+
+  def hashed_sentences
+    sentences.map { |sentence| JSON.parse(sentence) }
+  end
+end

app/models/analyze_syntax_response/sentence.rb

@@ -0,0 +1,15 @@
+module AnalyzeSyntaxResponse
+  class Sentence
+    include ActiveModel::Model
+
+    attr_accessor :text, :analyze_syntax_id
+
+    def begin_offset
+      text['beginOffset']
+    end
+
+    def content
+      text['content']
+    end
+  end
+end

app/models/analyze_syntax_response/token.rb

@@ -0,0 +1,22 @@
+module AnalyzeSyntaxResponse
+  class Token
+    include ActiveModel::Model
+
+    # rubocop:disable Style/SymbolLiteral, Naming/MethodName, Layout/EmptyLinesAroundAttributeAccessor
+    attr_accessor :text, :'partOfSpeech', :'dependencyEdge', :lemma, :analyze_syntax_id
+    # rubocop:enable Style/SymbolLiteral, Naming/MethodName, Layout/EmptyLinesAroundAttributeAccessor
+
+    def tag
+      # 戻り値は Google::Cloud::Language::V1::AnalyzeSyntaxResponse では Symbol だが、これは String である
+      part_of_speech['tag']
+    end
+
+    def part_of_speech
+      partOfSpeech
+    end
+
+    def dependency_edge
+      dependencyEdge
+    end
+  end
+end

app/models/tweet.rb

@@ -1,6 +1,7 @@
 class Tweet < ApplicationRecord
   has_paper_trail
 
+  has_one :analyze_syntax
   belongs_to :user
   has_many :assets
   has_many :hashtags

config/database.yml

@@ -1,7 +1,7 @@
 default: &default
   adapter: postgresql
   encoding: unicode
-  pool:     <%= ENV['RAILS_MAX_THREADS'] || Rails.application.credentials[:rails_max_threads] || 5 %>
+  pool: <%= ENV['RAILS_MAX_THREADS'] || Rails.application.credentials[:rails_max_threads] || 5 %>
 
 production:
   <<: *default

db/migrate/20210607214306_create_analyze_syntaxes.rb

@@ -0,0 +1,15 @@
+class CreateAnalyzeSyntaxes < ActiveRecord::Migration[6.1]
+  def change
+    create_table :analyze_syntaxes do |t|
+      t.string :language
+
+      # https://googleapis.dev/ruby/google-cloud-language-v1/latest/Google/Cloud/Language/V1/AnalyzeSyntaxResponse.html
+      t.text :sentences, array: true # レスポンスの生ログを保存する目的
+      t.text :tokens, array: true # レスポンスの生ログを保存する目的
+
+      t.references :tweet
+
+      t.timestamps
+    end
+  end
+end

spec/factories/analyze_syntaxes.rb

@@ -0,0 +1,28 @@
+FactoryBot.define do
+  factory :analyze_syntax do
+    language { 'ja' }
+    sentences {
+      [
+        "{\"text\":{\"content\":\"RT @foobar: オデッサを応援しています \\n#幻水総選挙運動\\n #幻水総選挙2021 https://t.co/3njNheDvPk\",\"beginOffset\":-1}}"
+      ]
+    }
+    tokens {
+      [
+        "{\"text\":{\"content\":\"オデッサ\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"NOUN\",\"proper\":\"PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":8,\"label\":\"DOBJ\"},\"lemma\":\"オデッサ\"}",
+        "{\"text\":{\"content\":\"を\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"PRT\",\"case\":\"ACCUSATIVE\",\"proper\":\"NOT_PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":6,\"label\":\"PRT\"},\"lemma\":\"を\"}",
+        "{\"text\":{\"content\":\"応援\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"NOUN\",\"proper\":\"NOT_PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":8,\"label\":\"ROOT\"},\"lemma\":\"応援\"}",
+        "{\"text\":{\"content\":\"し\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"VERB\",\"form\":\"GERUND\",\"proper\":\"NOT_PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":8,\"label\":\"MWV\"},\"lemma\":\"する\"}",
+        "{\"text\":{\"content\":\"て\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"PRT\",\"proper\":\"NOT_PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":8,\"label\":\"PRT\"},\"lemma\":\"て\"}",
+        "{\"text\":{\"content\":\"い\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"VERB\",\"form\":\"GERUND\",\"proper\":\"NOT_PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":8,\"label\":\"AUXVV\"},\"lemma\":\"い\"}",
+        "{\"text\":{\"content\":\"ます\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"VERB\",\"form\":\"ADNOMIAL\",\"proper\":\"NOT_PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":8,\"label\":\"AUX\"},\"lemma\":\"ます\"}",
+        "{\"text\":{\"content\":\"#\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"X\",\"proper\":\"NOT_PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":17,\"label\":\"NN\"},\"lemma\":\"#\"}",
+        "{\"text\":{\"content\":\"幻\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"NOUN\",\"proper\":\"PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":15,\"label\":\"NN\"},\"lemma\":\"幻\"}",
+        "{\"text\":{\"content\":\"水\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"NOUN\",\"proper\":\"PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":17,\"label\":\"NN\"},\"lemma\":\"水\"}",
+        "{\"text\":{\"content\":\"総\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"AFFIX\",\"proper\":\"PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":17,\"label\":\"PREF\"},\"lemma\":\"総\"}",
+        "{\"text\":{\"content\":\"選挙\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"NOUN\",\"proper\":\"NOT_PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":18,\"label\":\"NN\"},\"lemma\":\"選挙\"}",
+        "{\"text\":{\"content\":\"運動\",\"beginOffset\":-1},\"partOfSpeech\":{\"tag\":\"NOUN\",\"proper\":\"NOT_PROPER\"},\"dependencyEdge\":{\"headTokenIndex\":23,\"label\":\"NN\"},\"lemma\":\"運動\"}",
+      ]
+    }
+    # TODO: tweet_id { Tweet の Factory を作る }
+  end
+end

spec/models/analyze_syntax_spec.rb

@@ -0,0 +1,40 @@
+require 'rails_helper'
+
+RSpec.describe AnalyzeSyntax, type: :model do
+  let(:analyze_syntax) { build(:analyze_syntax) }
+
+  let(:hashed_sentences) { analyze_syntax.hashed_sentences }
+  let(:hashed_tokens) { analyze_syntax.hashed_tokens }
+
+  let(:tokens) { analyze_syntax.convert_analyze_syntax_response_token_objects }
+  let(:sentences) { analyze_syntax.convert_analyze_syntax_response_sentence_objects }
+
+  describe "#hashed_tokens" do
+    it 'tokens が Array in Hash で戻ってくること' do
+      expect(hashed_tokens.instance_of?(Array)).to be_truthy
+      expect(hashed_tokens.map(&:class).all? { |klass| klass == Hash }).to be_truthy
+    end
+  end
+
+  describe "#hashed_sentences" do
+    it 'sentences が Array in Hash で戻ってくること' do
+      expect(hashed_sentences.instance_of?(Array)).to be_truthy
+      expect(hashed_sentences.map(&:class).all? { |klass| klass == Hash }).to be_truthy
+    end
+  end
+
+  describe "#convert_analyze_syntax_response_token_objects" do
+    it 'tokens が Array in AnalyzeSyntaxResponse::Token で戻ってくること' do
+      expect(tokens.instance_of?(Array)).to be_truthy
+      expect(tokens.map(&:class).all? { |klass| klass == AnalyzeSyntaxResponse::Token }).to be_truthy
+    end
+  end
+
+  describe "#convert_analyze_syntax_response_sentence_objects" do
+    it 'sentences が Array in AnalyzeSyntaxResponse::Sentence で戻ってくること' do
+      expect(sentences.instance_of?(Array)).to be_truthy
+      expect(sentences.map(&:class).all? { |klass| klass == AnalyzeSyntaxResponse::Sentence }).to be_truthy
+    end
+  end
+end
+

spec/models/asset_spec.rb

@@ -1,5 +1,4 @@
 require 'rails_helper'
 
 RSpec.describe Asset, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
 end

spec/models/direct_message_spec.rb

@@ -1,5 +1,4 @@
 require 'rails_helper'
 
 RSpec.describe DirectMessage, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
 end

spec/models/hashtag_spec.rb

@@ -1,5 +1,4 @@
 require 'rails_helper'
 
 RSpec.describe Hashtag, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
 end

spec/models/in_tweet_url_spec.rb

@@ -1,5 +1,4 @@
 require 'rails_helper'
 
 RSpec.describe InTweetUrl, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
 end

spec/models/mention_spec.rb

@@ -1,5 +1,4 @@
 require 'rails_helper'
 
 RSpec.describe Mention, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
 end

spec/models/tweet_spec.rb

@@ -1,5 +1,4 @@
 require 'rails_helper'
 
 RSpec.describe Tweet, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
 end

spec/models/user_spec.rb

@@ -1,5 +1,4 @@
 require 'rails_helper'
 
 RSpec.describe User, type: :model do
-  pending "add some examples to (or delete) #{__FILE__}"
 end