From 0360f5a427ed6d88ebdcf9a2101d4b6070f7ad07 Mon Sep 17 00:00:00 2001 From: Ravi Yadav Date: Fri, 10 May 2019 13:22:16 +0530 Subject: [PATCH 1/3] Added Google Storage Adapter --- lib/sitemap_generator.rb | 17 ++++++----- .../adapters/google_storage_adapter.rb | 30 +++++++++++++++++++ sitemap_generator.gemspec | 1 + 3 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 lib/sitemap_generator/adapters/google_storage_adapter.rb diff --git a/lib/sitemap_generator.rb b/lib/sitemap_generator.rb index ab387e6b..5b52f6d9 100644 --- a/lib/sitemap_generator.rb +++ b/lib/sitemap_generator.rb @@ -7,14 +7,15 @@ require 'sitemap_generator/sitemap_location' module SitemapGenerator - autoload(:Interpreter, 'sitemap_generator/interpreter') - autoload(:FileAdapter, 'sitemap_generator/adapters/file_adapter') - autoload(:S3Adapter, 'sitemap_generator/adapters/s3_adapter') - autoload(:AwsSdkAdapter, 'sitemap_generator/adapters/aws_sdk_adapter') - autoload(:WaveAdapter, 'sitemap_generator/adapters/wave_adapter') - autoload(:FogAdapter, 'sitemap_generator/adapters/fog_adapter') - autoload(:BigDecimal, 'sitemap_generator/core_ext/big_decimal') - autoload(:Numeric, 'sitemap_generator/core_ext/numeric') + autoload(:Interpreter, 'sitemap_generator/interpreter') + autoload(:FileAdapter, 'sitemap_generator/adapters/file_adapter') + autoload(:S3Adapter, 'sitemap_generator/adapters/s3_adapter') + autoload(:AwsSdkAdapter, 'sitemap_generator/adapters/aws_sdk_adapter') + autoload(:WaveAdapter, 'sitemap_generator/adapters/wave_adapter') + autoload(:FogAdapter, 'sitemap_generator/adapters/fog_adapter') + autoload(:GoogleStorageAdapter, 'sitemap_generator/adapters/google_storage_adapter') + autoload(:BigDecimal, 'sitemap_generator/core_ext/big_decimal') + autoload(:Numeric, 'sitemap_generator/core_ext/numeric') SitemapError = Class.new(StandardError) SitemapFullError = Class.new(SitemapError) diff --git a/lib/sitemap_generator/adapters/google_storage_adapter.rb b/lib/sitemap_generator/adapters/google_storage_adapter.rb new file mode 100644 index 00000000..2f5d7419 --- /dev/null +++ b/lib/sitemap_generator/adapters/google_storage_adapter.rb @@ -0,0 +1,30 @@ +if !defined?(Google::Cloud::Storage) + raise "Error: `Google::Cloud::Storage` is not defined.\n\n"\ + "Please `require 'google/cloud/storage'` - or another library that defines this class." +end + +module SitemapGenerator + # Class for uploading sitemaps to a Google Storage supported endpoint. + class GoogleStorageAdapter + # Requires Google::Cloud::Storage to be defined. + # + # @param [Hash] opts Fog configuration options + # @option :credentials [Hash] Path to the google service account keyfile.json + # @option :project_id [String] Google Accounts project_id where the storage bucket resides + # @option :bucket [String] Name of Google Storage Bucket where the file is to be uploaded + def initialize(opts = {}) + @credentials = opts[:keyfile] || ENV[''] + @project_id = opts[:project_id] || ENV[''] + @bucket = opts[:bucket] || ENV[''] + end + + # Call with a SitemapLocation and string data + def write(location, raw_data) + SitemapGenerator::FileAdapter.new.write(location, raw_data) + + storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials) + bucket = storage.bucket @bucket + bucket.create_file location.path, location.path_in_public, acl: 'public' + end + end +end diff --git a/sitemap_generator.gemspec b/sitemap_generator.gemspec index 42fdd517..fe456594 100644 --- a/sitemap_generator.gemspec +++ b/sitemap_generator.gemspec @@ -18,5 +18,6 @@ Gem::Specification.new do |s| s.add_development_dependency 'rake' s.add_development_dependency 'aws-sdk-core' s.add_development_dependency 'aws-sdk-s3' + s.add_development_dependency 'google-cloud-storage' s.files = Dir.glob('{lib,rails,templates}/**/*') + %w(CHANGES.md MIT-LICENSE README.md VERSION) end From 6286deedd355a6b8811123427783afc44486e665 Mon Sep 17 00:00:00 2001 From: Ravi Yadav Date: Fri, 10 May 2019 14:48:07 +0530 Subject: [PATCH 2/3] Adding spec for GoogleStorageAdapter --- .../adapters/google_storage_adapter.rb | 20 ++++++++------- .../adapters/google_storage_adapter_spec.rb | 25 +++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 spec/sitemap_generator/adapters/google_storage_adapter_spec.rb diff --git a/lib/sitemap_generator/adapters/google_storage_adapter.rb b/lib/sitemap_generator/adapters/google_storage_adapter.rb index 2f5d7419..c1fbf83f 100644 --- a/lib/sitemap_generator/adapters/google_storage_adapter.rb +++ b/lib/sitemap_generator/adapters/google_storage_adapter.rb @@ -4,25 +4,27 @@ end module SitemapGenerator - # Class for uploading sitemaps to a Google Storage supported endpoint. + # Class for uploading sitemaps to a Google Storage using google-cloud-storage gem. class GoogleStorageAdapter # Requires Google::Cloud::Storage to be defined. # - # @param [Hash] opts Fog configuration options - # @option :credentials [Hash] Path to the google service account keyfile.json - # @option :project_id [String] Google Accounts project_id where the storage bucket resides - # @option :bucket [String] Name of Google Storage Bucket where the file is to be uploaded + # Options: + # :credentials [String] Path to the google service account keyfile.json + # :project_id [String] Google Accounts project_id where the storage bucket resides + # :bucket [String] Name of Google Storage Bucket where the file is to be uploaded + + # @param [Hash] opts Google::Cloud::Storage configuration options def initialize(opts = {}) - @credentials = opts[:keyfile] || ENV[''] - @project_id = opts[:project_id] || ENV[''] - @bucket = opts[:bucket] || ENV[''] + @credentials = opts[:keyfile] || ENV['GOOGLE_CLOUD_PROJECT'] + @project_id = opts[:project_id] || ENV['GOOGLE_APPLICATION_CREDENTIALS'] + @bucket = opts[:bucket] end # Call with a SitemapLocation and string data def write(location, raw_data) SitemapGenerator::FileAdapter.new.write(location, raw_data) - storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials) + storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials) bucket = storage.bucket @bucket bucket.create_file location.path, location.path_in_public, acl: 'public' end diff --git a/spec/sitemap_generator/adapters/google_storage_adapter_spec.rb b/spec/sitemap_generator/adapters/google_storage_adapter_spec.rb new file mode 100644 index 00000000..10e9242b --- /dev/null +++ b/spec/sitemap_generator/adapters/google_storage_adapter_spec.rb @@ -0,0 +1,25 @@ +# encoding: UTF-8 +require 'spec_helper' +require 'google/cloud/storage' + +describe SitemapGenerator::GoogleStorageAdapter do + let(:location) { SitemapGenerator::SitemapLocation.new } + let(:options) { {:credentials=>nil, :project_id=>nil} } + let(:google_bucket) { 'bucket' } + let(:adapter) { SitemapGenerator::GoogleStorageAdapter.new(options.merge(bucket: google_bucket)) } + + describe 'write' do + it 'it writes the raw data to a file and then uploads that file to Google Storage' do + bucket = double(:bucket) + storage = double(:storage) + bucket_resource = double(:bucket_resource) + expect(Google::Cloud::Storage).to receive(:new).with(options).and_return(storage) + expect(storage).to receive(:bucket).with('bucket').and_return(bucket_resource) + expect(location).to receive(:path_in_public).and_return('path_in_public') + expect(location).to receive(:path).and_return('path') + expect(bucket_resource).to receive(:create_file).with('path', 'path_in_public', acl: 'public').and_return(nil) + expect_any_instance_of(SitemapGenerator::FileAdapter).to receive(:write).with(location, 'raw_data') + adapter.write(location, 'raw_data') + end + end +end From 0d3d9a2238e1e3ba18d72554015d1e4275e7c386 Mon Sep 17 00:00:00 2001 From: Ravi Yadav Date: Fri, 10 May 2019 14:58:57 +0530 Subject: [PATCH 3/3] Updating Readme.md with usage of GoogleStorageAdapter --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index 37978f8b..3c0be48f 100644 --- a/README.md +++ b/README.md @@ -374,6 +374,27 @@ directory. Some documentation exists [on the wiki page][remote_hosts]. +* `SitemapGenerator::GoogleStorageAdapter` + + Uses `Google::Cloud::Storage` to upload to Google Cloud storage. + + You must `require 'google-cloud-storage'` in your sitemap config before using this adapter. + + An example of using this adapter in your sitemap configuration with options: + + ```ruby + SitemapGenerator::Sitemap.adapter = SitemapGenerator::GoogleStorageAdapter.new( + keyfile: 'path/to/keyfile.json', + project_id: 'google_account_project_id', + bucket: 'name_of_bucket' + ) + ``` + Also, inline with Google Authentication options, it can also pick credentials from environment variables. Variables required to be set for proper authentication with google are `GOOGLE_CLOUD_PROJECT` and `GOOGLE_APPLICATION_CREDENTIALS`. An example of using this adapter with the environment variables is: + + ```ruby + SitemapGenerator::Sitemap.adapter = SitemapGenerator::GoogleStorageAdapter.new( bucket: 'name_of_bucket' ) + ``` + #### An Example of Using an Adapter 1. Please see [this wiki page][remote_hosts] for more information about setting up SitemapGenerator to upload to a