Add memory-mapped utilility module #189

weiliw-amz · 2022-12-07T18:37:33Z

Issue #, if available:
N/A

Description of changes:
Add memory-mapped utilility module.

User could test with below code: Copy it into a filetest_mmap_util.cpp placed at pecos/core/util/, and run:

gcc -lm -ldl -lstdc++ -fopenmp -std=c++14 -lgcc -lgomp -O3  -I ./ test_mmap_util.cpp
./a.out

Output:

Generate a Bar with data:
---Bar---
---Foo---
foo_1: 0 1 2 3 4 5 6 7 8 9 
foo_2: 1
---------
bar: 5 5 5 5 5 
---------
Save Bar into mmap file: ./bar_test_mmap.txt
Load a new Bar from saved mmap file...
Loaded Bar data:
---Bar---
---Foo---
foo_1: 0 1 2 3 4 5 6 7 8 9 
foo_2: 1
---------
bar: 5 5 5 5 5 
---------

#include <iostream>
#include "mmap_util.hpp"

using namespace pecos::mmap_util;

// Nested class mmap example
// Bar contains a Foo instance
class Foo {
    public:
        Foo() {}
        ~Foo() { foo_1.clear(); }

        void init_data() {
            foo_1.resize(10, 0);
            for (int i=0; i<foo_1.size(); ++i) { foo_1[i] = i; }
            foo_2 = 1.0;
        }

        void print() {
            std::cout << "---Foo---" << std::endl;
            std::cout << "foo_1: ";
            for (int i=0; i<foo_1.size(); ++i) { std::cout << foo_1[i] << " "; }
            std::cout << std::endl;
            std::cout << "foo_2: " << foo_2 << std::endl;
            std::cout << "---------" << std::endl;
        }

        void save_to_mmap_store(MmapStore& mmap_s) const {
            foo_1.save_to_mmap_store(mmap_s);
            mmap_s.fput_one<double>(foo_2);
        }

        void load_from_mmap_store(MmapStore& mmap_s) {
            foo_1.load_from_mmap_store(mmap_s);
            foo_2 = mmap_s.fget_one<double>();
        }

    private:
        MmapableVector<int> foo_1;
        double foo_2;
};

class Bar {
    public:
        Bar() { }
        ~Bar() { bar.clear(); mmap_store.close(); }

        void init_data() {
            foo.init_data();
            bar.resize(5, 0);
            for (int i=0; i<bar.size(); ++i) { bar[i] = 5.0; }
        }

        void print() {
            std::cout << "---Bar---" << std::endl;
            foo.print();
            std::cout << "bar: ";
            for (int i=0; i<bar.size(); ++i) { std::cout << bar[i] << " "; }
            std::cout << std::endl;
            std::cout << "---------" << std::endl;
        }

        void save(const std::string & file_name) const {
            // Create a mmapfile for dump at the most outer layer class
            // You cannot reuse (i.e, close and reopen) mmap_store, since it may hold the data storage
            MmapStore mmap_s = MmapStore();
            mmap_s.open(file_name, "w");

            save_to_mmap_store(mmap_s);

            // Metadata dump and fp closure is automatically done at MmapStore destructor when this function ends
            // You can make it happen earlier with explicitly calling close()
            mmap_s.close();
        }
        void load(const std::string & file_name, const bool pre_load=true) {
            mmap_store.open(file_name, pre_load?"r":"r_lazy");
            load_from_mmap_store(mmap_store);
        }

        void save_to_mmap_store(MmapStore& mmap_s) const {
            foo.save_to_mmap_store(mmap_s);
            bar.save_to_mmap_store(mmap_s);
        }

        void load_from_mmap_store(MmapStore& mmap_s) {
            foo.load_from_mmap_store(mmap_s);
            bar.load_from_mmap_store(mmap_s);
        }

    private:
        Foo foo;
        MmapableVector<double> bar;
        // Mmap Data storage at the most outer layer class
        MmapStore mmap_store;
};


int main() {
    std::string f_name = "./bar_test_mmap.txt";

    std::cout << "Generate a Bar with data:" << std::endl;
    Bar bar;
    bar.init_data();
    bar.print();

    std::cout << "Save Bar into mmap file: " << f_name << std::endl;
    bar.save(f_name);

    std::cout << "Load a new Bar from saved mmap file..." << std::endl;
    Bar new_bar;
    new_bar.load(f_name, "r");

    std::cout << "Loaded Bar data:" << std::endl;
    new_bar.print();

    return 0;
}

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.

jiong-zhang · 2022-12-12T21:52:46Z