backport: seven zip ruby as cli wrapper

app/services/decidim/download_your_data_exporter.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "decidim/seven_zip_wrapper"
+
+module Decidim
+  # Public: Generates a 7z(seven zip) file with data files ready to be persisted
+  # somewhere so users can download their data.
+  #
+  # In fact, the 7z file wraps a ZIP file which finally contains the data files.
+  class DownloadYourDataExporter
+    DEFAULT_EXPORT_FORMAT = "CSV"
+    ZIP_FILE_NAME = "download-your-data.zip"
+
+    # Public: Initializes the class.
+    #
+    # user          - The user to export the data from.
+    # path          - The String path where to write the zip file.
+    # password      - The password to protect the zip file.
+    # export_format - The format of the data files inside the zip file. (CSV by default)
+    def initialize(user, path, password, export_format = DEFAULT_EXPORT_FORMAT)
+      @user = user
+      @path = File.expand_path path
+      @export_format = export_format
+      @password = password
+    end
+
+    def export
+      tmpdir = Dir.mktmpdir("temporary-download-your-data-dir")
+      user_data, user_attachments = data_and_attachments_for_user
+      save_user_data(tmpdir, user_data)
+      save_user_attachments(tmpdir, user_attachments)
+
+      SevenZipWrapper.compress_and_encrypt(filename: @path, password: @password, input_directory: tmpdir)
+    end
+
+    private
+
+    attr_reader :user, :export_format
+
+    def data_and_attachments_for_user
+      export_data = []
+      export_attachments = []
+      download_your_data_entities.each do |object|
+        klass = Object.const_get(object)
+        export_data << [klass.model_name.name.parameterize.pluralize, Exporters.find_exporter(export_format).new(klass.user_collection(user), klass.export_serializer).export]
+        attachments = klass.download_your_data_images(user)
+        export_attachments << [klass.model_name.name.parameterize.pluralize, attachments.flatten] unless attachments.nil?
+      end
+
+      [export_data, export_attachments]
+    end
+
+    def download_your_data_entities
+      @download_your_data_entities ||= DownloadYourDataSerializers.data_entities
+    end
+
+    def save_user_data(tmpdir, user_data)
+      user_data.each do |entity, exporter_data|
+        next if exporter_data.read == "\n"
+
+        file_name = File.join(tmpdir, "#{entity}-#{exporter_data.filename}")
+        File.write(file_name, exporter_data.read)
+      end
+    end
+
+    def save_user_attachments(tmpdir, user_attachments)
+      user_attachments.each do |entity, attachment_block|
+        attachment_block.each do |attachment|
+          next unless attachment.attached?
+
+          blobs = attachment.is_a?(ActiveStorage::Attached::One) ? [attachment.blob] : attachment.blobs
+          blobs.each do |blob|
+            Dir.mkdir(File.join(tmpdir, entity.parameterize))
+            file_name = File.join(tmpdir, entity.parameterize, blob.filename.to_s)
+            blob.open do |blob_file|
+              File.write(file_name, blob_file.read.force_encoding("UTF-8"))
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/decidim/seven_zip_wrapper.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "shellwords"
+
+module Decidim
+  class SevenZipWrapper
+    class << self
+      def compress_and_encrypt(filename:, password:, input_directory:)
+        run("cd #{escape(input_directory)} && 7z a -tzip -p#{escape(password)} -mem=AES256 #{escape(filename)} .")
+      end
+
+      def extract_and_decrypt(filename:, password:, output_directory:)
+        run("7z x -tzip #{escape(filename)} -o#{escape(output_directory)} -p#{escape(password)}")
+      end
+
+      private
+
+      def run(command)
+        success = system(command)
+
+        raise "Command failed: #{command}" unless success
+      end
+
+      def escape(string)
+        Shellwords.escape(string)
+      end
+    end
+  end
+end

spec/services/decidim/download_your_data_exporter_spec.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "decidim/seven_zip_wrapper"
+
+module Decidim
+  describe DownloadYourDataExporter do
+    subject { DownloadYourDataExporter.new(user, tmp_file_in, password) }
+
+    let(:tmp_file_in) do
+      Dir::Tmpname.create(["download-your-data", ".7z"]) do
+        # just get an empty file name
+      end
+    end
+    let(:tmp_dir_out) { Dir.mktmpdir("download_your_data_exporter_spec") }
+    let(:password) { "download-your-data.7z>passwd" }
+    let(:organization) { create(:organization) }
+    let(:user) { create :user, organization: organization }
+    let(:expected_files) do
+      # this are the prefixes for the files that could have user generated content
+      %w(
+        decidim-follows-
+        decidim-identities-
+        decidim-messaging-conversations-
+        decidim-notifications-
+        decidim-participatoryspaceprivateusers-
+        decidim-reports-
+        decidim-users-
+        decidim-usergroups-
+        decidim-meetings-registrations-
+        decidim-proposals-proposals-
+        decidim-budgets-orders-
+        decidim-forms-answers-
+        decidim-debates-debates-
+        decidim-conferences-conferenceregistrations-
+        decidim-conferences-conferenceinvites-
+        decidim-comments-comments-
+        decidim-comments-commentvotes-
+      )
+    end
+
+    describe "#export" do
+      it "compresses a password protected file" do
+        expect(File.exist?(tmp_file_in)).to be false
+
+        # generate 7z
+        subject.export
+
+        expect(File.exist?(tmp_file_in)).to be true
+
+        open_7z_and_extract_zip(tmp_file_in)
+
+        expect(Dir.entries(tmp_dir_out).count).to eq 4
+      end
+    end
+
+    describe "#data_and_attachments_for_user" do
+      it "returns an array of data for the user" do
+        user_data, = subject.send(:data_and_attachments_for_user)
+
+        file_prefixes = expected_files.dup
+        user_data.each do |entity, exporter_data|
+          entity_prefix = file_prefixes.find { |prefix| prefix.start_with?(entity) }
+          expect(file_prefixes.delete(entity_prefix)).to be_present
+
+          # we have an empty file  for each entity except for decidim-users
+          expect(exporter_data.read).to eq("\n") unless entity == "decidim-users"
+        end
+        expect(file_prefixes).to be_empty
+      end
+
+      context "when the user has a comment" do
+        let(:participatory_space) { create(:participatory_process, organization: organization) }
+        let(:component) { create(:component, participatory_space: participatory_space) }
+        let(:commentable) { create(:dummy_resource, component: component) }
+        let!(:comment) { create(:comment, commentable: commentable, author: user) }
+
+        it "returns the comment data" do
+          user_data, = subject.send(:data_and_attachments_for_user)
+
+          user_data.find { |entity, _| entity == "decidim-comments-comments" }.tap do |_, exporter_data|
+            csv_comments = exporter_data.read.split("\n")
+            expect(csv_comments.count).to eq 2
+            expect(csv_comments.first).to start_with "id;created_at;body;locale;author/id;author/name;alignment;depth;"
+            expect(csv_comments.second).to start_with "#{comment.id};"
+          end
+        end
+      end
+    end
+
+    private
+
+    def open_7z_and_extract_zip(file_path)
+      SevenZipWrapper.extract_and_decrypt(filename: file_path, password: password, output_directory: tmp_dir_out)
+    end
+  end
+end