From 1a0f840a8da77df7c89476eaef9aca7109d5bd4f Mon Sep 17 00:00:00 2001 From: Joe Sylve Date: Tue, 25 Jun 2019 13:59:45 -0500 Subject: [PATCH 1/5] Modernization refactorization --- aff4/Makefile.am | 4 +- aff4/aff4_directory.cc | 10 +- aff4/aff4_image.cc | 25 ++-- aff4/aff4_imager_utils.cc | 24 ++-- aff4/aff4_imager_utils.h | 64 +++++---- aff4/aff4_io.h | 4 +- aff4/aff4_map.cc | 9 +- aff4/aff4_registry.h | 61 -------- aff4/attributes.h | 293 ++++++++++++++++++++++++++++++++++++++ aff4/data_store.cc | 260 ++++++++++++++------------------- aff4/data_store.h | 101 +++++++------ aff4/lexicon.cc | 4 +- aff4/lexicon.h | 10 +- aff4/lexicon.inc | 2 +- aff4/libaff4-c.cc | 9 +- aff4/libaff4.cc | 19 +-- aff4/rdf.cc | 43 +++--- aff4/rdf.h | 111 ++++++--------- aff4/volume_group.cc | 36 +++-- aff4/zip.cc | 32 +++-- configure.ac | 10 ++ tests/aff4_map_tests.cc | 10 +- tests/data_store_test.cc | 8 +- tests/rdfquery_test.cc | 9 +- tools/pmem/linux_pmem.cc | 4 +- tools/pmem/osxpmem.cc | 6 +- tools/pmem/osxpmem.h | 16 +-- tools/pmem/pmem.h | 16 +-- tools/pmem/pmem_imager.cc | 6 +- tools/pmem/win_pmem.cc | 17 ++- tools/pmem/win_pmem.h | 20 +-- 31 files changed, 715 insertions(+), 528 deletions(-) delete mode 100644 aff4/aff4_registry.h create mode 100644 aff4/attributes.h diff --git a/aff4/Makefile.am b/aff4/Makefile.am index d6e126b..309e67e 100644 --- a/aff4/Makefile.am +++ b/aff4/Makefile.am @@ -48,7 +48,6 @@ libaff4_include_HEADERS = \ rdf.h\ aff4_errors.h \ aff4_init.h \ - aff4_registry.h \ aff4_utils.h \ data_store.h \ libaff4.h \ @@ -61,7 +60,8 @@ libaff4_include_HEADERS = \ aff4_symstream.h \ libaff4-c.h \ volume_group.h \ - threadpool.h + threadpool.h \ + attributes.h libaff4_la_SOURCES = \ aff4_image.cc \ diff --git a/aff4/aff4_directory.cc b/aff4/aff4_directory.cc index 177ed1d..9564052 100644 --- a/aff4/aff4_directory.cc +++ b/aff4/aff4_directory.cc @@ -67,10 +67,10 @@ AFF4Status AFF4Directory::NewAFF4Directory( RETURN_IF_ERROR(MkDir(resolver, root_path.c_str())); } - resolver->Set(new_obj->urn, AFF4_TYPE, new URN(AFF4_DIRECTORY_TYPE), + resolver->Set(new_obj->urn, AFF4_TYPE, URN(AFF4_DIRECTORY_TYPE), /* replace= */ false); - resolver->Set(new_obj->urn, AFF4_STORED, new URN(URN::NewURNFromFilename(root_path, true))); + resolver->Set(new_obj->urn, AFF4_STORED, URN::NewURNFromFilename(root_path, true)); result = std::move(new_obj); @@ -95,9 +95,9 @@ AFF4Status AFF4Directory::CreateMemberStream( std::string filename = member_name_for_urn(child, urn, false); // We are allowed to create any files inside the directory volume. - resolver->Set(child, AFF4_TYPE, new URN(AFF4_FILE_TYPE)); - resolver->Set(child, AFF4_STORED, new URN(urn)); - resolver->Set(child, AFF4_DIRECTORY_CHILD_FILENAME, new XSDString(filename)); + resolver->Set(child, AFF4_TYPE, URN(AFF4_FILE_TYPE)); + resolver->Set(child, AFF4_STORED, URN(urn)); + resolver->Set(child, AFF4_DIRECTORY_CHILD_FILENAME, XSDString(filename)); // Store the member inside our storage location. std::string full_path = root_path + PATH_SEP_STR + filename; diff --git a/aff4/aff4_image.cc b/aff4/aff4_image.cc index ef60a8c..d42dff5 100644 --- a/aff4/aff4_image.cc +++ b/aff4/aff4_image.cc @@ -494,11 +494,11 @@ AFF4Status AFF4Image::NewAFF4Image( new_obj->urn = image_urn; new_obj->current_volume = volume; - resolver->Set(image_urn, AFF4_TYPE, new URN(AFF4_IMAGESTREAM_TYPE), + resolver->Set(image_urn, AFF4_TYPE, URN(AFF4_IMAGESTREAM_TYPE), /* replace = */ false); - resolver->Set(image_urn, AFF4_STORED, new URN(volume->urn)); + resolver->Set(image_urn, AFF4_STORED, volume->urn); if(!resolver->HasURNWithAttribute(image_urn, AFF4_STREAM_SIZE)) { - resolver->Set(image_urn, AFF4_STREAM_SIZE, new XSDInteger((uint64_t)0)); + resolver->Set(image_urn, AFF4_STREAM_SIZE, XSDInteger(0ULL)); } new_obj = std::move(result); @@ -541,7 +541,7 @@ AFF4Status AFF4Image::FlushBevy() { RETURN_IF_ERROR(bevy_member->WriteStream(&bevy_stream, &empty_progress)); // Done with this bevy - make a new writer. - bevy_writer.reset(new _BevyWriter( + bevy_writer.reset(new _BevyWriter( // "new" needed here due to custom _BevyWriter deleter resolver, compression, chunk_size, chunks_per_segment)); bevy_number++; @@ -556,8 +556,8 @@ AFF4Status AFF4Image::Write(const char* data, size_t length) { // Prepare a bevy writer to collect the first bevy. if (bevy_writer == nullptr) { - bevy_writer.reset(new _BevyWriter(resolver, compression, chunk_size, - chunks_per_segment)); + bevy_writer.reset(new _BevyWriter(resolver, // "new" needed here due to custom _BevyWriter deleter + compression, chunk_size, chunks_per_segment)); } // This object is now dirty. @@ -882,18 +882,17 @@ AFF4Status AFF4Image::ReadBuffer(char* data, size_t* length) { } AFF4Status AFF4Image::_write_metadata() { - resolver->Set(urn, AFF4_TYPE, new URN(AFF4_IMAGESTREAM_TYPE), + resolver->Set(urn, AFF4_TYPE, URN(AFF4_IMAGESTREAM_TYPE), /* replace = */ false); - resolver->Set(urn, AFF4_STREAM_CHUNK_SIZE, new XSDInteger(chunk_size)); + resolver->Set(urn, AFF4_STREAM_CHUNK_SIZE, XSDInteger(chunk_size)); resolver->Set(urn, AFF4_STREAM_CHUNKS_PER_SEGMENT, - new XSDInteger(chunks_per_segment)); + XSDInteger(chunks_per_segment)); - resolver->Set(urn, AFF4_STREAM_SIZE, new XSDInteger(size)); + resolver->Set(urn, AFF4_STREAM_SIZE, XSDInteger(size)); - resolver->Set(urn, AFF4_IMAGE_COMPRESSION, new URN( - CompressionMethodToURN(compression))); + resolver->Set(urn, AFF4_IMAGE_COMPRESSION, CompressionMethodToURN(compression)); return STATUS_OK; } @@ -925,7 +924,7 @@ AFF4Status AFF4Image::Flush() { std::string AFF4Image::_FixupBevyData(std::string* data){ const uint32_t index_size = data->length() / sizeof(uint32_t); - std::unique_ptr bevy_index_array(new BevyIndex[index_size]); + auto bevy_index_array = absl::make_unique(index_size); uint32_t* bevy_index_data = reinterpret_cast(&(*data)[0]); uint64_t cOffset = 0; diff --git a/aff4/aff4_imager_utils.cc b/aff4/aff4_imager_utils.cc index 6b2fb92..3a9b015 100644 --- a/aff4/aff4_imager_utils.cc +++ b/aff4/aff4_imager_utils.cc @@ -122,7 +122,7 @@ AFF4Status BasicImager::ParseArgs() { if (Get("threads")->isSet()) { int threads = GetArg>("threads")->getValue(); resolver.logger->info("Will use {} threads.", threads); - resolver.pool.reset(new ThreadPool(threads)); + resolver.pool = absl::make_unique(threads); } // Check for incompatible commands. @@ -257,9 +257,7 @@ AFF4Status BasicImager::handle_aff4_volumes() { } AFF4Status BasicImager::handle_list() { - URN image_type(AFF4_IMAGE_TYPE); - for (const auto& subject: resolver.Query( - URN(AFF4_TYPE), &image_type)) { + for (const URN& subject: resolver.Query(AFF4_TYPE, URN(AFF4_IMAGE_TYPE))) { std::cout << subject.SerializeToString() << "\n"; } @@ -327,8 +325,7 @@ AFF4Status BasicImager::process_input() { image_urn.Set(volume->urn.Append(input_stream->urn.Path())); // Store the original filename. - resolver.Set(image_urn, AFF4_STREAM_ORIGINAL_FILENAME, - new XSDString(input)); + resolver.Set(image_urn, AFF4_STREAM_ORIGINAL_FILENAME, XSDString(input)); } // For very small streams, it is more efficient to just store them without @@ -353,7 +350,7 @@ AFF4Status BasicImager::process_input() { // Make this stream as an Image (Should we have // another type for a LogicalImage? - resolver.Set(segment->urn, AFF4_TYPE, new URN(AFF4_IMAGE_TYPE), + resolver.Set(segment->urn, AFF4_TYPE, URN(AFF4_IMAGE_TYPE), /* replace = */ false); // We need to explicitly check the abort status here. @@ -379,7 +376,7 @@ AFF4Status BasicImager::process_input() { // Make this stream as an Image (Should we have // another type for a LogicalImage? - resolver.Set(image_urn, AFF4_TYPE, new URN(AFF4_IMAGE_TYPE), + resolver.Set(image_urn, AFF4_TYPE, URN(AFF4_IMAGE_TYPE), /* replace = */ false); } } @@ -414,11 +411,8 @@ AFF4Status BasicImager::handle_export() { } } else { // These are all acceptable stream types. - for (const URN image_type : std::vector{ - URN(AFF4_IMAGE_TYPE), - URN(AFF4_MAP_TYPE) - }) { - for (const URN& image: resolver.Query(AFF4_TYPE, &image_type)) { + for (const URN &image_type : std::vector{ URN(AFF4_IMAGE_TYPE), URN(AFF4_MAP_TYPE)}) { + for (const URN& image: resolver.Query(AFF4_TYPE, image_type)) { if (aff4::fnmatch( export_pattern.c_str(), image.SerializeToString().c_str()) == 0) { @@ -561,10 +555,10 @@ AFF4Status BasicImager::GetOutputVolumeURN(URN* volume_urn) { resolver.logger->warn("Output file {} will be truncated.", output_volume_backing_urn); resolver.Set(output_volume_backing_urn, - AFF4_STREAM_WRITE_MODE, new XSDString("truncate")); + AFF4_STREAM_WRITE_MODE, XSDString("truncate")); } else { resolver.Set(output_volume_backing_urn, - AFF4_STREAM_WRITE_MODE, new XSDString("append")); + AFF4_STREAM_WRITE_MODE, XSDString("append")); } // The output is a directory volume. diff --git a/aff4/aff4_imager_utils.h b/aff4/aff4_imager_utils.h index 1276795..651e8fb 100644 --- a/aff4/aff4_imager_utils.h +++ b/aff4/aff4_imager_utils.h @@ -131,8 +131,10 @@ class BasicImager { // We use a list here to preserve insertion order. std::list> args; - void AddArg(TCLAP::Arg* arg) { - args.push_back(std::unique_ptr(arg)); + template + auto AddArg(Args && ...arg) + -> absl::enable_if_t::value, void> { + args.emplace_back(absl::make_unique(std::forward(arg)...)); } TCLAP::Arg* Get(std::string name) { @@ -174,22 +176,22 @@ class BasicImager { virtual AFF4Status Initialize(); virtual AFF4Status RegisterArgs() { - AddArg(new TCLAP::SwitchArg("V", "view", "View AFF4 metadata", false)); - AddArg(new TCLAP::SwitchArg("l", "list", "List all image streams in the volume.", false)); - AddArg(new TCLAP::MultiSwitchArg( - "d", "debug", "Display debugging logging (repeat for more info)", - false)); + AddArg("V", "view", "View AFF4 metadata", false); - AddArg(new TCLAP::SwitchArg( - "v", "verbose", "Display more verbose information", false)); + AddArg("l", "list", "List all image streams in the volume.", false); - AddArg(new TCLAP::SwitchArg( + AddArg("d", "debug", + "Display debugging logging (repeat for more info)", false); + + AddArg("v", "verbose", "Display more verbose information", false); + + AddArg( "t", "truncate", "Truncate the output file. Normally volumes and " "images are appended to existing files, but this flag forces the " "output file to be truncated first.", - false)); + false); - AddArg(new TCLAP::MultiArgToNextFlag( + AddArg( "i", "input", "File to image. If specified we copy these file to the " "output volume located at --output. If there is no AFF4 volume on " "--output yet, we create a new volume on it. A filename of @ means " @@ -197,14 +199,14 @@ class BasicImager { "occur). \n" "This can be specified multiple times with shell expansion. e.g.:\n" "-i /bin/*", - false, "/path/to/file/or/device")); + false, "/path/to/file/or/device"); - AddArg(new TCLAP::SwitchArg( + AddArg( "", "relative", "If specified all files will be written relative to " "the current directory. By default we write all files' absolute paths", - false)); + false); - AddArg(new TCLAP::ValueArg( + AddArg>( "e", "export", "Name of the streams to export. If specified we try " "to open this stream and write it to the export directory. Note that " "you will also need to specify an AFF4 volume path to load so we know " @@ -212,43 +214,43 @@ class BasicImager { "implies a stream residing in a loaded volume. E.g.\n" " -e /dev/sda -D /tmp/ my_volume.aff4", - false, "", "string")); + false, "", "string"); - AddArg(new TCLAP::ValueArg( + AddArg>( "", "logfile", "Specify a file to store log messages to", - false, "", "string")); + false, "", "string"); - AddArg(new TCLAP::ValueArg( + AddArg>( "D", "export_dir", "Directory to export to (Defaults to current " "directory)", - false, ".", "path to directory")); + false, ".", "path to directory"); - AddArg(new TCLAP::ValueArg( + AddArg>( "o", "output", "Output file to write to. If the file does not " "exist we create it. NOTE: If output is a directory we write an " "AFF4 Directory volume when imaging. If the output is '-' we write " "to stdout.", false, "", - "/path/to/file")); + "/path/to/file"); - AddArg(new TCLAP::SizeArg( + AddArg( "s", "split", "Split output volumes at this size.", false, 0, - "Size (E.g. 100Mb)")); + "Size (E.g. 100Mb)"); - AddArg(new TCLAP::ValueArg( + AddArg>( "c", "compression", "Type of compression to use (default deflate).", - false, "", "deflate, snappy, lz4, none")); + false, "", "deflate, snappy, lz4, none"); - AddArg(new TCLAP::ValueArg( + AddArg>( "", "threads", "Total number of threads to use.", - false, 2, "(default 2)")); + false, 2, "(default 2)"); - AddArg(new TCLAP::UnlabeledMultiArg( + AddArg>( "aff4_volumes", "These AFF4 Volumes will be loaded and their metadata will " "be parsed before the program runs.\n" "Note that this is necessary before you can extract streams with the " "--export flag.", - false, "/path/to/aff4/volume")); + false, "/path/to/aff4/volume"); return STATUS_OK; } diff --git a/aff4/aff4_io.h b/aff4/aff4_io.h index e1e22e4..2b89c94 100644 --- a/aff4/aff4_io.h +++ b/aff4/aff4_io.h @@ -30,6 +30,8 @@ specific language governing permissions and limitations under the License. #include "aff4/aff4_utils.h" #include "aff4/rdf.h" +#include + // A constant for various buffers used by the AFF4 library. #define AFF4_BUFF_SIZE (32 * 1024) @@ -200,7 +202,7 @@ class StringIO: public AFF4Stream { // Convenience constructors. static std::unique_ptr NewStringIO() { - return std::unique_ptr(new StringIO()); + return absl::make_unique(); } std::string Read(size_t length) override; diff --git a/aff4/aff4_map.cc b/aff4/aff4_map.cc index addfe68..27d346c 100644 --- a/aff4/aff4_map.cc +++ b/aff4/aff4_map.cc @@ -19,6 +19,7 @@ specific language governing permissions and limitations under the License. #include "aff4/volume_group.h" #include +#include #define BUFF_SIZE 1024*1024 @@ -64,9 +65,9 @@ AFF4Status AFF4Map::NewAFF4Map( new_object->last_target = data_stream; new_object->targets.push_back(data_stream); - resolver->Set(object_urn, AFF4_TYPE, new URN(AFF4_MAP_TYPE), + resolver->Set(object_urn, AFF4_TYPE, URN(AFF4_MAP_TYPE), /* replace= */ false); - resolver->Set(object_urn, AFF4_STORED, new URN(volume->urn)); + resolver->Set(object_urn, AFF4_STORED, URN(volume->urn)); new_object->current_volume = volume; result = std::move(new_object); @@ -115,7 +116,7 @@ AFF4Status AFF4Map::OpenAFF4Map( // Ensure the Range type hasn't added any extra data members static_assert(sizeof(BinaryRange) == sizeof(Range), "Range has been extended and must be converted here"); - auto buffer = std::unique_ptr{new Range[n]}; + auto buffer = absl::make_unique(n); map_stream->ReadIntoBuffer(buffer.get(), n * sizeof(BinaryRange)); @@ -514,7 +515,7 @@ AFF4Status AFF4Map::Flush() { } // Add the stream size property to the map - resolver->Set(urn, AFF4_STREAM_SIZE, new XSDInteger(size)); + resolver->Set(urn, AFF4_STREAM_SIZE, XSDInteger(size)); } return AFF4Stream::Flush(); diff --git a/aff4/aff4_registry.h b/aff4/aff4_registry.h deleted file mode 100644 index 0233244..0000000 --- a/aff4/aff4_registry.h +++ /dev/null @@ -1,61 +0,0 @@ -/* - Defines a library wide registration system for classes. -*/ -#ifndef SRC_AFF4_REGISTRY_H_ -#define SRC_AFF4_REGISTRY_H_ - -#include "aff4/config.h" - -#include -#include -#include -#include -#include - - -namespace aff4 { - - -class DataStore; -class URN; - -template -class ClassFactory { - public: - std::unique_ptr CreateInstance(const char* name, DataStore* resolver, - const URN* urn = nullptr) const { - return CreateInstance(std::string(name), resolver, urn); - } - - std::unique_ptr CreateInstance(std::string name, DataStore* data, - const URN* urn = nullptr) const { - T* instance = nullptr; - - // find name in the registry and call factory method. - const auto it = factoryFunctionRegistry.find(name); - if (it != factoryFunctionRegistry.end()) { - instance = it->second(data, urn); - } - - return std::unique_ptr(instance); - } - - void RegisterFactoryFunction( - std::string name, std::function< - T*(DataStore*, const URN*)> classFactoryFunction) - { - // register the class factory function - factoryFunctionRegistry[name] = classFactoryFunction; - } - - private: - std::unordered_map< - std::string, - std::function - > factoryFunctionRegistry; -}; - -#endif // SRC_AFF4_REGISTRY_H_ - - -} // namespace aff4 diff --git a/aff4/attributes.h b/aff4/attributes.h new file mode 100644 index 0000000..512e9b4 --- /dev/null +++ b/aff4/attributes.h @@ -0,0 +1,293 @@ +/* +Copyright 2019 BlackBag Technologies, Inc. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); you may not use +this file except in compliance with the License. You may obtain a copy of the +License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +*/ + +#pragma once + +#include "aff4/rdf.h" + +#include + +#include + +namespace aff4 { + +class AttributeValue { + // Stub + template + struct is_variant_member; + + /** + * Determines if type T is a valid member of a variant + */ + template + struct is_variant_member> + : public absl::disjunction, ALL_T>...> {}; + + // Stub + template + struct validate_base; + + /** + * Determines if all members of a variant have a base of type T + */ + template + struct validate_base> + : public absl::conjunction...> {}; + + /** + * A default constructable state to represent an uninitialized or + * invalid AttributeValue. + */ + struct invalid : public RDFValue{ + std::string SerializeToString() const final { + return ""; + } + + /** + * Invalid states aren't comparable + */ + constexpr bool operator==(const invalid &) const noexcept { + return false; + } + }; + + /** + * AttributeValue storage type + * + * To add additional storage types add them to this variant. + * Types must inherit from RDFValue and also need to be added + * to the type enum below (in order). + */ + using storage = absl::variant; + static_assert(validate_base::value, + "storage types must all inherit from RDFValue"); + + public: + /** + * AttributeValue storage types + * + * This enum must stay in sync with the types in the storage variant + */ + enum type { + INVALID, + RDFBytes, + XSDString, + MD5Hash, + SHA1Hash, + SHA256Hash, + SHA512Hash, + Blake2BHash, + XSDInteger, + XSDBoolean, + URN, + _COUNT_, // This must be last and is not a valid type + }; + static_assert(absl::variant_size::value == type::_COUNT_, + "storage types and type enum are out of sync"); + + /** + * Determines if type T is a valid AttributeValue type + */ + template + using is_valid_type = is_variant_member; + + /** + * Exception type that is thrown if trying to access + * the wrong AttributeValue type + */ + using bad_access = absl::bad_variant_access; + + public: + /** + * Constructs an AttributeValue in an invalid state + */ + constexpr AttributeValue() = default; + + /** + * Constructs an AttributeValue based on a member type + */ + template ::value>> + constexpr AttributeValue(T &&t) noexcept + : val{std::forward(t)} {} + + /** + * Returns the type of the AttributeValue + */ + constexpr type Type() const noexcept { + return static_cast(val.index()); + } + + /** + * Tests if the AttributeValue is of type t + */ + constexpr bool IsType(type t) const noexcept { + return (Type() == t); + } + + /** + * Test is the AttributeValue is of type T + * + * This overload is useful for template metaprogramming + */ + template ::value>> + constexpr bool IsType() const noexcept { + return absl::holds_alternative(val); + } + + /** + * Test whether the AttributeValue is in a valid state + */ + constexpr bool IsValid() const noexcept { + return !(IsType(INVALID) || val.valueless_by_exception()); + } + + /** + * Allows implicit casting to a boolean + * + * Returns true if the AttributeValue is in a valid state + */ + constexpr operator bool() const noexcept { + return IsValid(); + } + + /** + * Allows implicit casting to valid AttributeValue types. + * + * If the AttributeValue is not of type T, then an AttributeValue::bad_access + * exception is thrown. + */ + template ::value>> + constexpr operator const T&() const { + return absl::get(val); + } + + /** + * Allows implicit casting to valid AttributeValue type pointers. + * + * If the AttributeValue is not of type T, then nullptr is returned + */ + template ::value>> + constexpr operator const T*() const noexcept { + return absl::get_if(&val); + } + + /** + * Allows casting the AttributeValue to a RDFValue pointer + */ + operator const RDFValue *() const noexcept { + return absl::visit( + [](const RDFValue & base) { return &base; }, + val); + } + + /** + * Allows casting the AttributeValue to a RDFValue pointer + */ + operator RDFValue *() noexcept { + return absl::visit( + [](RDFValue & base) { return &base; }, + val); + } + + /** + * Provides convenient access to the AttributeValue's RDFValue members. + */ + const RDFValue * operator->() const noexcept { + return (*this); + } + + /** + * Provides convenient access to the AttributeValue's RDFValue members. + */ + RDFValue * operator->() noexcept { + return (*this); + } + + /** + * Allows comparision with valid AttributeValue types + */ + template ::value>> + constexpr bool operator==(const T& other) const noexcept { + return IsType() && (static_cast(*this) == other); + } + + /** + * Allows comparision with valid AttributeValue types + */ + template ::value>> + constexpr bool operator!=(const T& other) const noexcept { + return !operator==(other); + } + + /** + * Allows comparision with other Attributes + */ + constexpr bool operator==(const AttributeValue& other) const noexcept { + return (val == other.val); + } + + /** + * Allows comparision with other Attributes + */ + constexpr bool operator!=(const AttributeValue& other) const noexcept { + return !operator==(other); + } + + private: + storage val{}; // The base storage for the AttributeValue +}; + +/** + * A one-to-many set of attributes + */ +using AttributeSet = std::unordered_map>; + +class AttributeFactory { + public: + AttributeValue CreateInstance(const std::string & name) const { + // find name in the registry and call factory method. + const auto it = storage.find(name); + if (it == storage.end()) { + // Invalid type + return {}; + } + + return it->second; + } + + template::value>> + void Register(std::string && name) { + storage.emplace(std::forward(name), T{}); + } + + private: + std::unordered_map storage{}; +}; + +// A Global Registry for RDFValue. This factory will provide the correct +// RDFValue instance based on the turtle type URN. For example xsd:integer -> +// XSDInteger(). +extern AttributeFactory AttributeRegistry; + +} // namespace aff4 \ No newline at end of file diff --git a/aff4/data_store.cc b/aff4/data_store.cc index 1ed681a..8f38cf2 100644 --- a/aff4/data_store.cc +++ b/aff4/data_store.cc @@ -18,12 +18,15 @@ #include "aff4/lexicon.h" #include "aff4/data_store.h" #include "aff4/libaff4.h" +#include "aff4/attributes.h" #include #include #include #include #include "aff4/aff4_symstream.h" +#include + namespace aff4 { DataStore::DataStore(): @@ -65,7 +68,7 @@ bool DataStore::ShouldSuppress(const URN& subject, DataStore::DataStore(DataStoreOptions options) : logger(options.logger), - pool(std::unique_ptr(new ThreadPool(options.threadpool_size))) { + pool(absl::make_unique(options.threadpool_size)) { // Add these default namespace. namespaces.push_back(std::pair("aff4", AFF4_NAMESPACE)); @@ -118,7 +121,7 @@ class RaptorSerializer { static std::unique_ptr NewRaptorSerializer( URN base, const std::vector>& namespaces) { - std::unique_ptr result(new RaptorSerializer()); + std::unique_ptr result {new RaptorSerializer()}; // "new" needed here due to protected constructor raptor_uri* uri; result->world = raptor_world_pool.get(); @@ -140,7 +143,7 @@ class RaptorSerializer { } AFF4Status AddStatement(const URN& subject, const URN& predicate, - const RDFValue* value) { + const AttributeValue& value) { raptor_statement* triple = raptor_new_statement(world); triple->subject = raptor_new_term_from_uri_string( world, @@ -177,49 +180,42 @@ class RaptorSerializer { } }; -static std::unique_ptr RDFValueFromRaptorTerm(DataStore* resolver, raptor_term* term) { +static AttributeValue RDFValueFromRaptorTerm(raptor_term* term) { if (term->type == RAPTOR_TERM_TYPE_URI) { char* uri = reinterpret_cast(raptor_uri_to_string(term->value.uri)); - std::unique_ptr result(new URN(uri)); + AttributeValue result = URN(uri); raptor_free_memory(uri); return result; } + const std::string value_string(reinterpret_cast(term->value.literal.string), + term->value.literal.string_len); + if (term->type == RAPTOR_TERM_TYPE_LITERAL) { // Does it have a special data type? if (term->value.literal.datatype) { char* uri = reinterpret_cast(raptor_uri_to_string(term->value.literal.datatype)); - std::unique_ptr result = - RDFValueRegistry.CreateInstance(uri, resolver); + AttributeValue result = AttributeRegistry.CreateInstance(uri); + raptor_free_memory(uri); + // If we do not know how to handle this type we skip it. if (!result) { - raptor_free_memory(uri); - return nullptr; + return {}; } - raptor_free_memory(uri); - - std::string value_string( - reinterpret_cast(term->value.literal.string), - term->value.literal.string_len); - if (result->UnSerializeFromString(value_string) != STATUS_OK) { - return nullptr; + return {}; } return result; - - // No special type - this is just a string. - } else { - std::string value_string( - reinterpret_cast(term->value.literal.string), - term->value.literal.string_len); - - return std::unique_ptr(new XSDString(value_string)); } + + // No special type - this is just a string. + return XSDString(value_string); } - return nullptr; + + return {}; } static void statement_handler(void* user_data, raptor_statement* statement) { @@ -233,10 +229,8 @@ static void statement_handler(void* user_data, raptor_statement* statement) { char* predicate = reinterpret_cast( raptor_uri_to_string(statement->predicate->value.uri)); - std::unique_ptr object( - RDFValueFromRaptorTerm(resolver, statement->object)); - - if (object.get()) { + AttributeValue object = RDFValueFromRaptorTerm(statement->object); + if (object) { resolver->Set(URN(subject), URN(predicate), std::move(object), /* replace = */ false); } @@ -258,7 +252,7 @@ class RaptorParser { public: static std::unique_ptr NewRaptorParser(DataStore* resolver) { - std::unique_ptr result(new RaptorParser(resolver)); + std::unique_ptr result {new RaptorParser(resolver)}; // "new" needed here due to protected constructor result->world = raptor_world_pool.get(); @@ -316,11 +310,13 @@ AFF4Status MemoryDataStore::DumpToTurtle(AFF4Stream& output_stream, URN base, bo return MEMORY_ERROR; } - for (const auto& it : store) { - URN subject = it.first; + for (const auto it : store) { + const URN & subject = it.first; + const AttributeSet & attributes = it.second; - for (const auto& attr_it : it.second) { - URN predicate = attr_it.first; + for (const auto attr_it : attributes) { + const URN & predicate = attr_it.first; + const std::vector & values = attr_it.second; // Volatile predicates are suppressed. if (!verbose) { @@ -333,13 +329,12 @@ AFF4Status MemoryDataStore::DumpToTurtle(AFF4Stream& output_stream, URN base, bo } // Load all attributes. - for (const auto& a: attr_it.second) { - const RDFValue* value = a.get(); - + for (const auto& value: values) { // Skip this URN if it is in the suppressed_rdftypes set. - if (ShouldSuppress( - subject, predicate, value->SerializeToString())) + if (ShouldSuppress(subject, predicate, + value->SerializeToString())) { continue; + } serializer->AddStatement(subject, predicate, value); } @@ -372,198 +367,153 @@ AFF4Status MemoryDataStore::LoadFromTurtle(AFF4Stream& stream) { return STATUS_OK; } -void MemoryDataStore::Set(const URN& urn, const URN& attribute, RDFValue* value, +void MemoryDataStore::Set(const URN& urn, const URN& attribute, AttributeValue && value, bool replace) { - if (value == nullptr) abort(); - std::shared_ptr unique_value(value); - Set(urn, attribute, unique_value, replace); -} - -void MemoryDataStore::Set(const URN& urn, const URN& attribute, - std::shared_ptr value, bool replace) { - // Automatically create needed keys. - std::vector> values = store[urn.SerializeToString()][ - attribute.SerializeToString()]; - - if (replace) values.clear(); + std::vector & values = store[urn][attribute]; - values.push_back(std::move(value)); + if (replace) { + values.clear(); + } - store[urn.SerializeToString()][attribute.SerializeToString()] = values; + values.emplace_back(std::forward(value)); } AFF4Status MemoryDataStore::Get(const URN& urn, const URN& attribute, - RDFValue& value) { - auto urn_it = store.find(urn.SerializeToString()); - + AttributeValue& value) const { + const auto urn_it = store.find(urn); if (urn_it == store.end()) { return NOT_FOUND; } - auto attribute_itr = urn_it->second.find(attribute.SerializeToString()); - if (attribute_itr == urn_it->second.end()) { + const auto & attributes = urn_it->second; + + const auto attr_it = attributes.find(attribute); + if (attr_it == attributes.end()) { // Since the majority of AFF4 objects in practice are zip segments, // as an optimization we don't store type attributes for these // objects. Instead, objects without type attriutes are assumed to // be zip segments. if (attribute == AFF4_TYPE) { - value.UnSerializeFromString(AFF4_ZIP_SEGMENT_TYPE); + value = URN(AFF4_ZIP_SEGMENT_TYPE); return STATUS_OK; } return NOT_FOUND; } - std::vector> values = (attribute_itr->second); - AFF4Status res = NOT_FOUND; - for (const auto &fetched_value: values) { - // Only collect compatible types. - const RDFValue& fetched_value_ref = *fetched_value; - if (typeid(value) == typeid(fetched_value_ref)) { - res = value.UnSerializeFromString( - fetched_value->SerializeToString()); - } - } + const auto & values = attr_it->second; + value = values.front(); - return res; + return STATUS_OK; } -AFF4Status MemoryDataStore::Get(const URN& urn, - const URN& attribute, - std::vector>& values) { - auto urn_it = store.find(urn.SerializeToString()); - +AFF4Status MemoryDataStore::Get(const URN& urn, const URN& attribute, + std::vector& values) const { + const auto urn_it = store.find(urn); if (urn_it == store.end()) { return NOT_FOUND; } - auto attribute_itr = urn_it->second.find(attribute.SerializeToString()); - if (attribute_itr == urn_it->second.end()) { + const auto & attributes = urn_it->second; + + const auto attr_it = attributes.find(attribute); + + if (attr_it == attributes.end()) { // Since the majority of AFF4 objects in practice are zip segments, // as an optimization we don't store type attributes for these // objects. Instead, objects without type attriutes are assumed to // be zip segments. if (attribute == AFF4_TYPE) { - values.emplace_back(new URN(AFF4_ZIP_SEGMENT_TYPE)); + values.emplace_back(URN(AFF4_ZIP_SEGMENT_TYPE)); return STATUS_OK; } return NOT_FOUND; } - std::vector> ivalues = (attribute_itr->second); - if (ivalues.empty()) { - return NOT_FOUND; - } - // Load up our keys. - for(std::vector>::iterator it = ivalues.begin(); it != ivalues.end(); it++){ - std::shared_ptr v = *it; - values.push_back(v); - } + values = attr_it->second; + return STATUS_OK; } -bool MemoryDataStore::HasURN(const URN& urn) { - auto urn_it = store.find(urn.SerializeToString()); - - if (urn_it == store.end()) { - return false; - } - return true; +bool MemoryDataStore::HasURN(const URN& urn) const { + return (store.find(urn) != store.end()); } -bool MemoryDataStore::HasURNWithAttribute(const URN& urn, const URN& attribute) { - auto urn_it = store.find(urn.SerializeToString()); - +bool MemoryDataStore::HasURNWithAttribute(const URN& urn, const URN& attribute) const { + const auto urn_it = store.find(urn); if (urn_it == store.end()) { return false; } - auto attribute_itr = urn_it->second.find(attribute.SerializeToString()); - if (attribute_itr == urn_it->second.end()) { - return false; - } + const AttributeSet & attributes = urn_it->second; - std::vector> values = (attribute_itr->second); - if (values.empty()) { - return false; - } - return true; + return (attributes.find(attribute) != attributes.end()); } -bool MemoryDataStore::HasURNWithAttributeAndValue( - const URN& urn, const URN& attribute, const RDFValue& value) { - auto urn_it = store.find(urn.SerializeToString()); - std::string serialized_value = value.SerializeToString(); - +bool MemoryDataStore::HasURNWithAttributeAndValue(const URN& urn, const URN& attribute, + const AttributeValue& value) const { + const auto urn_it = store.find(urn); if (urn_it == store.end()) { return false; } - auto attribute_itr = urn_it->second.find(attribute.SerializeToString()); - if (attribute_itr == urn_it->second.end()) { - return false; - } + const AttributeSet & attributes = urn_it->second; - std::vector> values = (attribute_itr->second); - if (values.empty()) { + const auto attr_it = attributes.find(attribute); + if (attr_it == attributes.end()) { return false; } - // iterator over all values looking for a RDFValue that matches the attribute requested. - for (const auto& v: values) { - if (serialized_value == v->SerializeToString()) { - return true; - } - } + const std::vector & values = attr_it->second; - return false; + return (std::find(values.begin(), values.end(), value) != values.end()); } -std::unordered_set MemoryDataStore::Query( - const URN& attribute, const RDFValue* value) { - std::unordered_set results; - std::string serialized_value; - std::string serialized_attribute = attribute.SerializeToString(); +std::unordered_set MemoryDataStore::Query(const URN& attribute, + const AttributeValue & value) const { + std::unordered_set results{}; + + for (const auto it: store) { + const URN & subject = it.first; - if (value) { - serialized_value = value->SerializeToString(); - } + const AttributeSet & attributes = it.second; - for (const auto &it: store) { - URN subject = it.first; - AFF4_Attributes attr = it.second; + const auto attr_it = attributes.find(attribute); + if (attr_it == attributes.end()) { + // No candidate attribute found. Go to next subject + continue; + } - for (const auto &it: attr) { - URN stored_attribute = it.first; - if (stored_attribute.SerializeToString() != serialized_attribute) - continue; + if (!value.IsValid()) { + // We're not looking for a specific value, so add the subject + // to the result set + results.insert(subject); + continue; + } - const auto& value_array = it.second; + const std::vector & values = attr_it->second; - for (const auto &stored_value: value_array) { - if (value == nullptr || - serialized_value == stored_value->SerializeToString()) { - results.insert(subject); - } - } + const auto val_it = std::find(values.begin(), values.end(), value); + if (val_it != values.end()) { + // We've got a matching pair + results.insert(subject); } } return results; } -AFF4_Attributes MemoryDataStore::GetAttributes(const URN& urn) { - AFF4_Attributes attr; - if(!HasURN(urn)){ - return attr; +AttributeSet MemoryDataStore::GetAttributes(const URN& urn) const { + const auto urn_it = store.find(urn); + if (urn_it == store.end()) { + return {}; } - auto urn_it = store.find(urn.SerializeToString()); - attr = urn_it->second; - return attr; + + return urn_it->second; } AFF4Status MemoryDataStore::DeleteSubject(const URN& urn) { - store.erase(urn.SerializeToString()); + store.erase(urn); return STATUS_OK; } diff --git a/aff4/data_store.h b/aff4/data_store.h index 359af3b..400a212 100644 --- a/aff4/data_store.h +++ b/aff4/data_store.h @@ -31,7 +31,7 @@ specific language governing permissions and limitations under the License. #include "aff4/aff4_utils.h" #include -#include "aff4/rdf.h" +#include "aff4/attributes.h" namespace aff4 { @@ -42,9 +42,6 @@ class AFF4Volume; class DataStore; class AFF4SymbolicStream; -// AFF4_Attributes are a collection of RDFValue objects, keyed by attributes. -typedef std::unordered_map>> AFF4_Attributes; - // Deleter for AFF4Flusher struct AFF4Flusher_deleter { template @@ -121,32 +118,46 @@ class DataStore { std::unique_ptr pool; virtual void Set(const URN& urn, const URN& attribute, - RDFValue* value, bool replace = true) = 0; + AttributeValue && value, bool replace = true) = 0; virtual AFF4Status Get(const URN& urn, const URN& attribute, - RDFValue& value ) = 0; + AttributeValue& value ) const = 0; virtual AFF4Status Get(const URN& urn, const URN& attribute, - std::vector>& values) = 0; + std::vector& values) const = 0; + + template::value>> + AFF4Status Get(const URN& urn, const URN& attribute, T & value) const { + AttributeValue v; + AFF4Status ret = Get(urn, attribute, v); + if (ret != STATUS_OK) { + return ret; + } + + if (!v.IsType()) { + return INCOMPATIBLE_TYPES; + } + + value = v; + + return STATUS_OK; + } /** * Does the given URN have the given attribute set to the given value. */ - virtual bool HasURNWithAttributeAndValue( - const URN& urn, const URN& attribute, const RDFValue& value) = 0; + virtual bool HasURNWithAttributeAndValue(const URN& urn, const URN& attribute, + const AttributeValue& value) const = 0; /** * Does the given URN have the given attribute set */ - virtual bool HasURNWithAttribute(const URN& urn, const URN& attribute) = 0; + virtual bool HasURNWithAttribute(const URN& urn, const URN& attribute) const = 0; /** * Does the datastore know the given urn */ - virtual bool HasURN(const URN& urn) = 0; - - virtual void Set(const URN& urn, const URN& attribute, - std::shared_ptr value, - bool replace = true) = 0; + virtual bool HasURN(const URN& urn) const = 0; virtual AFF4Status DeleteSubject(const URN& urn) = 0; @@ -160,8 +171,8 @@ class DataStore { * @param value The optional value which to check against. * @return A vector of Resource URNs that have the given attribute. */ - virtual std::unordered_set Query( - const URN& attribute, const RDFValue* value = nullptr) = 0; + virtual std::unordered_set Query(const URN& attribute, + const AttributeValue & value = {}) const = 0; /** * Get the AFF4 Attributes for the given urn. @@ -170,7 +181,7 @@ class DataStore { * @return All known AFF4_Attributes for the given urn. If the urn * is unknown, an empty attribute list is returned. */ - virtual AFF4_Attributes GetAttributes(const URN& urn) = 0; + virtual AttributeSet GetAttributes(const URN& urn) const = 0; #ifdef AFF4_HAS_LIBYAML_CPP // Dump ourselves to a yaml file. @@ -223,7 +234,7 @@ class DataStore { class MemoryDataStore: public DataStore { private: // Store a collection of AFF4_Attributes at each URN. - std::unordered_map store; + std::unordered_map store; public: MemoryDataStore() = default; @@ -236,34 +247,44 @@ class MemoryDataStore: public DataStore { virtual ~MemoryDataStore(); - /** - * Set the RDFValue in the data store. Note that the data store will retain - * ownership of the value, and therefore callers may not use it after this - * call. - * - * @param urn: The subject to set the attribute for. - * @param attribute: The attribute to set. - * @param value: The value. - */ - virtual void Set(const URN& urn, const URN& attribute, RDFValue* value, - bool replace = true) override; + void Set(const URN& urn, const URN & attribute, AttributeValue && value, + bool replace = true) override; - virtual void Set(const URN& urn, const URN& attribute, - std::shared_ptr value, bool replace = true) override; + AFF4Status Get(const URN& urn, const URN& attribute, + AttributeValue& value ) const override; - AFF4Status Get(const URN& urn, const URN& attribute, RDFValue& value) override; AFF4Status Get(const URN& urn, const URN& attribute, - std::vector>& value) override; + std::vector& values) const override; + + template::value>> + AFF4Status Get(const URN& urn, const URN& attribute, T & value) const { + AttributeValue v; + AFF4Status ret = Get(urn, attribute, v); + if (ret != STATUS_OK) { + return ret; + } + + if (!v.IsType()) { + return INCOMPATIBLE_TYPES; + } + + value = v; + + return STATUS_OK; + } + + bool HasURN(const URN& urn) const override; + + bool HasURNWithAttribute(const URN& urn, const URN& attribute) const override; - bool HasURN(const URN& urn) override; - bool HasURNWithAttribute(const URN& urn, const URN& attribute) override; bool HasURNWithAttributeAndValue( - const URN& urn, const URN& attribute, const RDFValue& value) override; + const URN& urn, const URN& attribute, const AttributeValue& value) const override; - std::unordered_set Query( - const URN& attribute, const RDFValue* value = nullptr) override; + std::unordered_set Query(const URN& attribute, + const AttributeValue & value = {}) const override; - AFF4_Attributes GetAttributes(const URN& urn) override; + AttributeSet GetAttributes(const URN& urn) const override; AFF4Status DeleteSubject(const URN& urn) override; diff --git a/aff4/lexicon.cc b/aff4/lexicon.cc index 7bd500c..e3f4e8a 100644 --- a/aff4/lexicon.cc +++ b/aff4/lexicon.cc @@ -37,14 +37,14 @@ Schema Schema::GetSchema(std::string object_type) { // Define all the Schema and attributes. Schema AFF4_OBJECT_SCHEMA("object"); AFF4_OBJECT_SCHEMA.AddAttribute( - "type", Attribute( + "type", SchemaAttribute( AFF4_TYPE, URNType, "The type of this object.")); Schema AFF4_STREAM_SCHEMA("generic_stream"); AFF4_STREAM_SCHEMA.AddParent(AFF4_OBJECT_SCHEMA); AFF4_STREAM_SCHEMA.AddAttribute( - "size", Attribute( + "size", SchemaAttribute( AFF4_STREAM_SIZE, XSDIntegerType, "How large the object is in bytes.")); diff --git a/aff4/lexicon.h b/aff4/lexicon.h index 890dbc1..1a2fe22 100644 --- a/aff4/lexicon.h +++ b/aff4/lexicon.h @@ -69,7 +69,7 @@ URN CompressionMethodToURN(AFF4_IMAGE_COMPRESSION_ENUM method); * @param name * @param type */ -class Attribute { +class SchemaAttribute { protected: std::string name; std::string type; @@ -80,9 +80,9 @@ class Attribute { std::unordered_map allowed_values; public: - Attribute() {} + SchemaAttribute() {} - Attribute(std::string name, std::string type, std::string description): + SchemaAttribute(std::string name, std::string type, std::string description): name(name), type(type), description(description) {} void AllowedValue(std::string alias, std::string value) { @@ -98,7 +98,7 @@ class Attribute { */ class Schema { protected: - std::unordered_map attributes; + std::unordered_map attributes; std::string object_type; /// This schema inherits from these parents. @@ -111,7 +111,7 @@ class Schema { Schema(std::string object_type): object_type(object_type) {} - void AddAttribute(std::string alias, Attribute attribute) { + void AddAttribute(std::string alias, SchemaAttribute attribute) { attributes[alias] = attribute; } diff --git a/aff4/lexicon.inc b/aff4/lexicon.inc index 6b38c90..5ebd4df 100644 --- a/aff4/lexicon.inc +++ b/aff4/lexicon.inc @@ -36,7 +36,7 @@ LEXICON_DEFINE(XSDIntegerTypeInt, "http://www.w3.org/2001/XMLSchema#int"); LEXICON_DEFINE(XSDIntegerTypeLong, "http://www.w3.org/2001/XMLSchema#long"); LEXICON_DEFINE(XSDBooleanType, "http://www.w3.org/2001/XMLSchema#boolean"); -/// Attribute names for different AFF4 objects. +/// SchemaAttribute names for different AFF4 objects. /// Base AFF4Object LEXICON_DEFINE(AFF4_TYPE, "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"); diff --git a/aff4/libaff4-c.cc b/aff4/libaff4-c.cc index c2576bb..39359ab 100644 --- a/aff4/libaff4-c.cc +++ b/aff4/libaff4-c.cc @@ -24,6 +24,8 @@ #include "aff4/libaff4.h" #include "aff4/libaff4-c.h" +#include + class LogHandler { public: @@ -122,12 +124,11 @@ struct AFF4_Handle { // Attempt AFF4 Standard, and if not, fallback to AFF4 Evimetry Legacy format. - const aff4::URN type(aff4::AFF4_IMAGE_TYPE); - auto images = resolver.Query(aff4::AFF4_TYPE, &type); + auto images = resolver.Query(aff4::AFF4_TYPE, aff4::URN(aff4::AFF4_IMAGE_TYPE)); if (images.empty()) { const aff4::URN legacy_type(aff4::AFF4_LEGACY_IMAGE_TYPE); - images = resolver.Query(aff4::URN(aff4::AFF4_TYPE), &legacy_type); + images = resolver.Query(aff4::URN(aff4::AFF4_TYPE), aff4::URN(aff4::AFF4_LEGACY_IMAGE_TYPE)); if (images.empty()) { return false; } @@ -163,7 +164,7 @@ class HandlePool { return it->release(); } - std::unique_ptr h(new AFF4_Handle(filename)); + auto h = absl::make_unique(filename); if (h->open()) { return h.release(); diff --git a/aff4/libaff4.cc b/aff4/libaff4.cc index 0a6bead..823c18c 100644 --- a/aff4/libaff4.cc +++ b/aff4/libaff4.cc @@ -35,6 +35,8 @@ specific language governing permissions and limitations under the License. #include #include +#include + namespace aff4 { // Flip to true to immediately stop operations. @@ -279,7 +281,7 @@ std::string aff4_sprintf(std::string fmt, ...) { int size = fmt.size() * 2 + 50; while (1) { - std::unique_ptr buffer(new char[size + 1]); + auto buffer = absl::make_unique(size + 1); // Null terminate the buffer (important on MSVC which does not always // terminate). @@ -307,22 +309,21 @@ int AFF4Stream::sprintf(std::string fmt, ...) { int size = fmt.size() * 2 + 50; while (1) { - char* buffer = new char[size + 1]; + auto buffer = absl::make_unique(size + 1); // Null terminate the buffer (important on MSVC which does not always // terminate). buffer[size] = 0; va_start(ap, fmt); - int n = vsnprintf(buffer, size, fmt.c_str(), ap); + int n = vsnprintf(buffer.get(), size, fmt.c_str(), ap); va_end(ap); if (n > -1 && n < size) { // Everything worked - Write(buffer, n); - delete[] buffer; + Write(buffer.get(), n); return n; } - delete[] buffer; + if (n > -1) { // Needed size returned size = n + 1; // For null char } else { @@ -374,12 +375,6 @@ aff4_off_t AFF4Volume::Size() const { } -ClassFactory* GetAFF4ClassFactory() { - static auto* factory = new ClassFactory(); - return factory; -} - - #ifdef _WIN32 std::string GetLastErrorMessage() { diff --git a/aff4/rdf.cc b/aff4/rdf.cc index b985311..4637e72 100644 --- a/aff4/rdf.cc +++ b/aff4/rdf.cc @@ -14,6 +14,7 @@ specific language governing permissions and limitations under the License. */ #include "aff4/aff4_base.h" +#include "aff4/attributes.h" #include "aff4/lexicon.h" #include "aff4/rdf.h" #include @@ -24,6 +25,8 @@ specific language governing permissions and limitations under the License. #include #include +#include + #ifdef _WIN32 #include #endif @@ -259,11 +262,6 @@ std::string URN::Domain() const { return ""; } -URN::URN(const char* data): URN() { - value = std::string(data); -} - - raptor_term* URN::GetRaptorTerm(raptor_world* world) const { std::string value_string(SerializeToString()); @@ -279,8 +277,7 @@ URN URN::Append(const std::string& component) const { i--; } - const std::string urn = value.substr(0, i+1) + _NormalizePath(component); - return URN(urn); + return {value.substr(0, i+1) + _NormalizePath(component)}; } @@ -349,7 +346,7 @@ static std::string abspath(std::string path) { // The windows version of this function is somewhat simpler. DWORD buffer_len = GetFullPathName(path.c_str(), 0, NULL, NULL); if (buffer_len > 0) { - auto buffer = std::unique_ptr(new TCHAR[buffer_len]); + auto buffer = absl::make_unique(buffer_len); GetFullPathName(path.c_str(), buffer_len, buffer.get(), NULL); return std::string(buffer.get()); } @@ -370,7 +367,7 @@ std::string URN::ToFilename() const { } const int bytesNeeded = std::max(value.size() + 1, (size_t)MAX_PATH); - auto path = std::unique_ptr(new char[bytesNeeded]); + auto path = absl::make_unique(bytesNeeded); DWORD path_length = bytesNeeded; HRESULT res; @@ -409,7 +406,7 @@ static std::string abspath(std::string path) { // Prepend the CWD to the path. int path_len = PATH_MAX; while (1) { - std::unique_ptr cwd (new char[path_len]); + auto cwd = absl::make_unique(path_len); // Try again with a bigger size. if (nullptr==getcwd(cwd.get(), path_len) && errno == ERANGE) { @@ -426,7 +423,7 @@ static std::string abspath(std::string path) { // Unix version to ToFilename(). std::string URN::ToFilename() const { const int bytesNeeded = value.size() + 1; - std::unique_ptr path (new char[bytesNeeded]); + auto path = absl::make_unique(bytesNeeded); if (uriUriStringToUnixFilenameA(value.c_str(), path.get()) != URI_SUCCESS) { return ""; @@ -445,7 +442,7 @@ URN URN::NewURNFromOSFilename(std::string filename, bool windows_filename, filename = abspath(filename); } - char* tmp = new char[filename.size() * 3 + 8 + 1]; + auto tmp = absl::make_unique(filename.size() * 3 + 8 + 1); /* Windows filename -> URL handling is pretty complex. The urlparser library * does a reasonable job but misses some important edge cases. Microsoft @@ -470,17 +467,17 @@ URN URN::NewURNFromOSFilename(std::string filename, bool windows_filename, #endif { if (uriWindowsFilenameToUriStringA( - filename.c_str(), tmp) == URI_SUCCESS) { - result.value = std::string(tmp); + filename.c_str(), tmp.get()) == URI_SUCCESS) { + result.value = std::string(tmp.get()); } }; // Unix filename - } else if (uriUnixFilenameToUriStringA(filename.c_str(), tmp) == + } else if (uriUnixFilenameToUriStringA(filename.c_str(), tmp.get()) == URI_SUCCESS) { - result.value = std::string(tmp); + result.value = std::string(tmp.get()); } - delete[] tmp; + return result; } @@ -580,11 +577,19 @@ raptor_term* XSDBoolean::GetRaptorTerm(raptor_world* world) const { return result; } - // A Global Registry for RDFValue. This factory will provide the correct // RDFValue instance based on the turtle type URN. For example xsd:integer -> // XSDInteger(). -ClassFactory RDFValueRegistry; +AttributeFactory AttributeRegistry{}; + +template::value>> +class RDFValueRegistrar { + public: + RDFValueRegistrar(std::string && name) { + AttributeRegistry.Register(std::forward(name)); + } +}; static RDFValueRegistrar r1(RDFBytesType); diff --git a/aff4/rdf.h b/aff4/rdf.h index d9f0529..888e5fa 100644 --- a/aff4/rdf.h +++ b/aff4/rdf.h @@ -23,7 +23,6 @@ specific language governing permissions and limitations under the License. // #include "aff4/config.h" #include "aff4/aff4_errors.h" -#include "aff4/aff4_registry.h" #include "aff4/aff4_utils.h" namespace aff4 { @@ -49,12 +48,15 @@ class URN; * */ class RDFValue { - protected: - DataStore* resolver; - public: - explicit RDFValue(DataStore* resolver): resolver(resolver) {} - RDFValue(): resolver(nullptr) {} + RDFValue() = default; + + RDFValue(const RDFValue&) = default; + RDFValue & operator=(const RDFValue&) = default; + + RDFValue(RDFValue&&) = default; + RDFValue & operator=(RDFValue&&) = default; + virtual ~RDFValue() {} virtual raptor_term* GetRaptorTerm(raptor_world* world) const { @@ -63,9 +65,7 @@ class RDFValue { } // RDFValues must provide methods for serializing and unserializing. - virtual std::string SerializeToString() const { - return ""; - } + virtual std::string SerializeToString() const = 0; virtual AFF4Status UnSerializeFromString(const char* data, int length) { UNUSED(data); @@ -86,25 +86,6 @@ class RDFValue { // Support direct logging of RDFValues (like URNs etc.) std::ostream& operator<<(std::ostream& os, const RDFValue& c); -// A Global Registry for RDFValue. This factory will provide the correct -// RDFValue instance based on the turtle type URN. For example xsd:integer -> -// XSDInteger(). -extern ClassFactory RDFValueRegistry; - -template -class RDFValueRegistrar { - public: - explicit RDFValueRegistrar(std::string name) { - // register the class factory function - RDFValueRegistry.RegisterFactoryFunction( - name, - [](DataStore *resolver, const URN *urn) -> RDFValue * { - UNUSED(urn); - return new T(resolver); - }); - } -}; - static const char* const lut = "0123456789ABCDEF"; @@ -115,17 +96,17 @@ static const char* const lut = "0123456789ABCDEF"; */ class RDFBytes: public RDFValue { public: - std::string value; + std::string value{}; - explicit RDFBytes(std::string data): - RDFValue(), value(data) {} + RDFBytes(const std::string & data) : value(data) {} - RDFBytes(const char* data, unsigned int length): - RDFValue(), value(data, length) {} + RDFBytes(std::string && data) + : value(std::forward(data)) {} - explicit RDFBytes(DataStore* resolver): RDFValue(resolver) {} + RDFBytes(const char* data, unsigned int length) + : value(data, length) {} - RDFBytes() {} + RDFBytes() = default; std::string SerializeToString() const; AFF4Status UnSerializeFromString(const char* data, int length); @@ -146,15 +127,13 @@ class RDFBytes: public RDFValue { */ class XSDString: public RDFBytes { public: - XSDString(std::string data): - RDFBytes(data.c_str(), data.size()) {} - - XSDString(const char* data): - RDFBytes(data, strlen(data)) {} + using RDFBytes::RDFBytes; - explicit XSDString(DataStore* resolver): RDFBytes(resolver) {} + XSDString(const char * const value) + : RDFBytes(value, std::strlen(value)) {} - XSDString() {} + template + XSDString(const char (&value)[N]) : RDFBytes(value, N) {} std::string SerializeToString() const; AFF4Status UnSerializeFromString(const char* data, int length); @@ -201,21 +180,25 @@ class Blake2BHash : public XSDString { */ class XSDInteger: public RDFValue { public: - uint64_t value; + uint64_t value{}; - explicit XSDInteger(uint64_t data): - RDFValue(nullptr), value(data) {} + explicit XSDInteger(uint64_t data) : value(data) {} - explicit XSDInteger(DataStore* resolver): - RDFValue(resolver), value(0) {} - - XSDInteger() : value(0){} + XSDInteger() = default; std::string SerializeToString() const; AFF4Status UnSerializeFromString(const char* data, int length); raptor_term* GetRaptorTerm(raptor_world* world) const; + + bool operator==(const XSDInteger& other) const { + return (value == other.value); + } + + bool operator==(uint64_t other) const { + return (value == other); + } }; @@ -225,21 +208,25 @@ class XSDInteger: public RDFValue { */ class XSDBoolean: public RDFValue { public: - bool value; - - explicit XSDBoolean(bool data): - RDFValue(nullptr), value(data) {} + bool value{}; - explicit XSDBoolean(DataStore* resolver): - RDFValue(resolver), value(false) {} + explicit XSDBoolean(bool data) : value(data) {} - XSDBoolean() : value(false){} + XSDBoolean() = default; std::string SerializeToString() const; AFF4Status UnSerializeFromString(const char* data, int length); raptor_term* GetRaptorTerm(raptor_world* world) const; + + bool operator==(const XSDBoolean& other) const { + return (value == other.value); + } + + bool operator==(bool other) const { + return (value == other); + } }; /** @@ -247,10 +234,9 @@ class XSDBoolean: public RDFValue { * */ class URN: public XSDString { - protected: - //std::string original_filename; - public: + using XSDString::XSDString; + /** * Create a new URN from a filename. * @@ -284,13 +270,6 @@ class URN: public XSDString { */ std::string ToFilename() const; - URN(const char* data); - URN(const std::string& data): URN(data.c_str()) {}; - explicit URN(DataStore* resolver): URN() { - UNUSED(resolver); - }; - URN() {}; - URN Append(const std::string& component) const; raptor_term* GetRaptorTerm(raptor_world* world) const; diff --git a/aff4/volume_group.cc b/aff4/volume_group.cc index 887b00c..cae71a5 100644 --- a/aff4/volume_group.cc +++ b/aff4/volume_group.cc @@ -16,13 +16,11 @@ void VolumeGroup::AddVolume(AFF4Flusher &&volume) { // Construct the appropriate stream and return it. AFF4Status VolumeGroup::GetStream(URN stream_urn, AFF4Flusher &result) { // Get all the type attrbutes of the URN. - std::vector> types; + std::vector types; if (STATUS_OK == resolver->Get(stream_urn, AFF4_TYPE, types)) { - for (auto &type : types) { - std::string type_str(type->SerializeToString()); - - if (type_str == AFF4_IMAGESTREAM_TYPE || - type_str == AFF4_LEGACY_IMAGESTREAM_TYPE) { + for (const URN &type : types) { + if (type == AFF4_IMAGESTREAM_TYPE || + type == AFF4_LEGACY_IMAGESTREAM_TYPE) { AFF4Flusher image_stream; RETURN_IF_ERROR( AFF4Image::OpenAFF4Image( @@ -31,7 +29,7 @@ AFF4Status VolumeGroup::GetStream(URN stream_urn, AFF4Flusher &resul result = std::move(image_stream); resolver->logger->debug("Openning {} as type {}", - stream_urn, type_str); + stream_urn, type.value); return STATUS_OK; } @@ -44,21 +42,21 @@ AFF4Status VolumeGroup::GetStream(URN stream_urn, AFF4Flusher &resul // regular stream with NewAFF4Image or NewAFF4Map and then set // the aff4:dataStream of a new object to a concerete Map or // ImageStream. - if (type_str == AFF4_IMAGE_TYPE || - type_str == AFF4_DISK_IMAGE_TYPE || - type_str == AFF4_VOLUME_IMAGE_TYPE || - type_str == AFF4_MEMORY_IMAGE_TYPE || - type_str == AFF4_CONTIGUOUS_IMAGE_TYPE || - type_str == AFF4_DISCONTIGUOUS_IMAGE_TYPE) { + if (type == AFF4_IMAGE_TYPE || + type == AFF4_DISK_IMAGE_TYPE || + type == AFF4_VOLUME_IMAGE_TYPE || + type == AFF4_MEMORY_IMAGE_TYPE || + type == AFF4_CONTIGUOUS_IMAGE_TYPE || + type == AFF4_DISCONTIGUOUS_IMAGE_TYPE) { + URN delegate; - if (STATUS_OK == resolver->Get(stream_urn, AFF4_DATASTREAM, delegate)) { // TODO: This can get recursive. Protect against abuse. return GetStream(delegate, result); } } - if (type_str == AFF4_MAP_TYPE) { + if (type == AFF4_MAP_TYPE) { AFF4Flusher map_stream; RETURN_IF_ERROR( AFF4Map::OpenAFF4Map( @@ -66,7 +64,7 @@ AFF4Status VolumeGroup::GetStream(URN stream_urn, AFF4Flusher &resul result = std::move(map_stream); resolver->logger->debug("Openning {} as type {}", - stream_urn, type_str); + stream_urn, type.value); return STATUS_OK; } @@ -74,12 +72,12 @@ AFF4Status VolumeGroup::GetStream(URN stream_urn, AFF4Flusher &resul // Zip segments are stored directly in each volume. We use // the resolver to figure out which volume has each // segment. - if (type_str == AFF4_ZIP_SEGMENT_TYPE || - type_str == AFF4_FILE_TYPE) { + if (type == AFF4_ZIP_SEGMENT_TYPE || + type == AFF4_FILE_TYPE) { URN owner; RETURN_IF_ERROR(resolver->Get(stream_urn, AFF4_STORED, owner)); - resolver->logger->debug("Openning {} as type {}", stream_urn, type_str); + resolver->logger->debug("Opening {} as type {}", stream_urn, type.value); auto it = volume_objs.find(owner); if (it != volume_objs.end()) { return (it->second->OpenMemberStream(stream_urn, result)); diff --git a/aff4/zip.cc b/aff4/zip.cc index 53dc909..2d2569a 100644 --- a/aff4/zip.cc +++ b/aff4/zip.cc @@ -30,6 +30,8 @@ #include "aff4/lexicon.h" #include "aff4/libaff4.h" +#include + namespace aff4 { @@ -62,9 +64,9 @@ AFF4Status ZipFile::NewZipFile( self->backing_stream = std::move(backing_stream); - resolver->Set(self->urn, AFF4_TYPE, new URN(AFF4_ZIP_TYPE), + resolver->Set(self->urn, AFF4_TYPE, URN(AFF4_ZIP_TYPE), /* replace = */ false); - resolver->Set(self->urn, AFF4_STORED, new URN(self->backing_stream->urn)); + resolver->Set(self->urn, AFF4_STORED, self->backing_stream->urn); // Mark the container with its URN. Some AFF4 implementations // rely on this being the first segment in the volume. @@ -107,8 +109,8 @@ AFF4Status ZipFile::LoadTurtleMetadata() { // Ensure the correct backing store URN overrides the one stored in the // turtle file since it is more current. - resolver->Set(urn, AFF4_STORED, new URN(backing_stream->urn)); - resolver->Set(backing_stream->urn, AFF4_CONTAINS, new URN(urn)); + resolver->Set(urn, AFF4_STORED, backing_stream->urn); + resolver->Set(backing_stream->urn, AFF4_CONTAINS, URN(urn)); return STATUS_OK; } @@ -213,10 +215,10 @@ AFF4Status ZipFile::parse_cd() { urn.Set(urn_string); // Set these triples so we know how to open the zip file again. - resolver->Set(urn, AFF4_TYPE, new URN(AFF4_ZIP_TYPE), + resolver->Set(urn, AFF4_TYPE, URN(AFF4_ZIP_TYPE), /*replace =*/ false); - resolver->Set(urn, AFF4_STORED, new URN(backing_stream->urn)); - resolver->Set(backing_stream->urn, AFF4_CONTAINS, new URN(urn)); + resolver->Set(urn, AFF4_STORED, backing_stream->urn); + resolver->Set(backing_stream->urn, AFF4_CONTAINS, urn); } } @@ -298,7 +300,7 @@ AFF4Status ZipFile::parse_cd() { return PARSING_ERROR; } - std::unique_ptr zip_info(new ZipInfo()); + auto zip_info = absl::make_unique(); zip_info->filename = backing_stream->Read(entry.file_name_length).c_str(); zip_info->local_header_offset = entry.relative_offset_local_header; @@ -352,7 +354,7 @@ AFF4Status ZipFile::parse_cd() { // Store this information in the resolver. Ths allows segments to be // directly opened by URN. URN member_urn = urn_from_member_name(zip_info->filename, urn); - resolver->Set(member_urn, AFF4_STORED, new URN(urn)); + resolver->Set(member_urn, AFF4_STORED, urn); members[zip_info->filename] = std::move(zip_info); } @@ -468,7 +470,7 @@ AFF4Status ZipFile::CreateMemberStream( URN segment_urn, AFF4Flusher &result) { - resolver->Set(segment_urn, AFF4_STORED, new URN(urn)); + resolver->Set(segment_urn, AFF4_STORED, urn); auto new_obj = make_flusher(resolver); new_obj->urn = segment_urn; @@ -556,7 +558,7 @@ AFF4Status ZipFileSegment::OpenZipFileSegment( // If the file is deflated we have no choice but to read it all into memory. case ZIP_DEFLATE: { unsigned int buffer_size = zip_info->file_size; - std::unique_ptr decomp_buffer(new char[buffer_size]); + auto decomp_buffer = absl::make_unique(buffer_size); std::string c_buffer = backing_store->Read(zip_info->compress_size); @@ -654,7 +656,7 @@ std::string ZipFileSegment::CompressBuffer( // Get an upper bound on the size of the compressed buffer. int buffer_size = deflateBound(&strm, buffer.size() + 10); - std::unique_ptr c_buffer(new char[buffer_size]); + auto c_buffer = absl::make_unique(buffer_size); strm.next_out = reinterpret_cast(c_buffer.get()); strm.avail_out = buffer_size; @@ -701,7 +703,7 @@ AFF4Status ZipFileSegment::Flush() { // Borrow a reference to the backing stream. auto backing_store = owner->backing_stream.get(); resolver->logger->debug("Writing member {}", urn); - std::unique_ptr zip_info(new ZipInfo()); + auto zip_info = absl::make_unique(); // Append member at the end of the file. if (backing_store->properties.seekable) { @@ -909,7 +911,7 @@ AFF4Status ZipFile::StreamAddMember(URN member_urn, AFF4Stream& stream, // zip_info offsets are relative to the start of the zip file (take // global_offset into account). - std::unique_ptr zip_info(new ZipInfo()); + auto zip_info = absl::make_unique(); zip_info->filename = member_name_for_urn(member_urn, urn, true); zip_info->local_header_offset = backing_stream->Tell() - global_offset; @@ -926,7 +928,7 @@ AFF4Status ZipFile::StreamAddMember(URN member_urn, AFF4Stream& stream, memset(&strm, 0, sizeof(strm)); // Make some room for output buffer. - std::unique_ptr c_buffer(new char[AFF4_BUFF_SIZE]); + auto c_buffer = absl::make_unique(AFF4_BUFF_SIZE); strm.next_out = reinterpret_cast(c_buffer.get()); strm.avail_out = AFF4_BUFF_SIZE; diff --git a/configure.ac b/configure.ac index 1914136..25db65e 100644 --- a/configure.ac +++ b/configure.ac @@ -67,6 +67,16 @@ AC_CHECK_LIB([pthread], [pthread_create], [], [AC_MSG_ERROR([pthread library not #Check for cpp header only libs PKG_CHECK_MODULES([TCLAP], [tclap], [], [AC_MSG_ERROR([The TCLAP flag parsing library (libtclap-dev) is not found])]) +# Check for abseil libraries +ABSEIL_LIBS=" -labsl_bad_variant_access " +LDFLAGS="$LDFLAGS $ABSEIL_LIBS" +AC_LINK_IFELSE( + [AC_LANG_PROGRAM([#include ], [absl::variant dummy;])], + [HAVE_ABSEIL=1], + [AC_MSG_ERROR([abseil library not found.])] +) + + # configure options. AC_ARG_WITH([yaml], [AS_HELP_STRING([--with-yaml], [Enable YAML support (default is no)])], [WITH_YAML=$withval], [WITH_YAML=no]) AC_ARG_ENABLE([static-binaries], diff --git a/tests/aff4_map_tests.cc b/tests/aff4_map_tests.cc index 5b95439..d686d74 100644 --- a/tests/aff4_map_tests.cc +++ b/tests/aff4_map_tests.cc @@ -67,13 +67,13 @@ class AFF4MapTest: public ::testing::Test { image_urn_streamed = image_urn.Append("streamed"); // First create a stream and add some data in it. - AFF4Flusher source(new StringIO(&resolver)); + auto source = make_flusher(&resolver); // Fill it with data. source->Write("AAAABBBBCCCCDDDDEEEEFFFFGGGGHHHH"); // Make a temporary map that defines our plan. - AFF4Flusher helper_map(new AFF4Map(&resolver)); + auto helper_map = make_flusher(&resolver); helper_map->AddRange(4, 0, 4, source.get()); // 0000AAAA helper_map->AddRange(0, 12, 4, source.get()); // DDDDAAAA @@ -102,10 +102,10 @@ TEST_F(AFF4MapTest, TestAddRange) { EXPECT_OK(NewFileBackedObject(&resolver, filename, "read", file)); EXPECT_OK(ZipFile::OpenZipFile(&resolver, std::move(file), zip)); - AFF4Flusher a(new StringIO(&resolver)); - AFF4Flusher b(new StringIO(&resolver)); + auto a = make_flusher(&resolver); + auto b = make_flusher(&resolver); - AFF4Flusher devnul(new StringIO(&resolver)); + auto devnul = make_flusher(&resolver); AFF4Flusher map; EXPECT_OK(AFF4Map::NewAFF4Map( diff --git a/tests/data_store_test.cc b/tests/data_store_test.cc index 0f67dd6..28be305 100644 --- a/tests/data_store_test.cc +++ b/tests/data_store_test.cc @@ -13,7 +13,7 @@ class MemoryDataStoreTest: public ::testing::Test { TEST_F(MemoryDataStoreTest, IncompatibleGet) { RDFBytes result; - store.Set(URN("hello"), URN("World"), new XSDString("foo")); + store.Set(URN("hello"), URN("World"), XSDString("foo")); // This should fail since the value is the wrong type. EXPECT_EQ(NOT_FOUND, @@ -24,14 +24,14 @@ TEST_F(MemoryDataStoreTest, IncompatibleGet) { TEST_F(MemoryDataStoreTest, StorageTest) { XSDString result; - store.Set(URN("hello"), URN("World"), new XSDString("foo")); + store.Set(URN("hello"), URN("World"), XSDString("foo")); EXPECT_EQ(STATUS_OK, store.Get(URN("hello"), URN("World"), result)); EXPECT_STREQ(result.SerializeToString().c_str(), "foo"); - store.Set(URN("hello"), URN("World"), new XSDString("bar")); + store.Set(URN("hello"), URN("World"), XSDString("bar")); // In the current implementation a second Set() overwrites the previous value. EXPECT_EQ(STATUS_OK, @@ -42,7 +42,7 @@ TEST_F(MemoryDataStoreTest, StorageTest) { TEST_F(MemoryDataStoreTest, TurtleSerializationTest) { - store.Set(URN("hello"), URN("World"), new XSDString("foo")); + store.Set(URN("hello"), URN("World"), XSDString("foo")); MemoryDataStore new_store; std::unique_ptr output = StringIO::NewStringIO(); diff --git a/tests/rdfquery_test.cc b/tests/rdfquery_test.cc index 913efbe..c1845a9 100644 --- a/tests/rdfquery_test.cc +++ b/tests/rdfquery_test.cc @@ -40,8 +40,7 @@ TEST_F(AFF4ImageRDFQuery, Sample1URN) { EXPECT_OK(ZipFile::OpenZipFile(&resolver, std::move(file), zip)); // Query the resolver. - const URN type(AFF4_IMAGE_TYPE); - std::unordered_set images = resolver.Query(URN(AFF4_TYPE), &type); + std::unordered_set images = resolver.Query(URN(AFF4_TYPE), URN(AFF4_IMAGE_TYPE)); ASSERT_EQ(1, images.size()); for(URN u : images){ @@ -61,8 +60,7 @@ TEST_F(AFF4ImageRDFQuery, Sample2URN) { EXPECT_OK(NewFileBackedObject(&resolver, filename, "read", file)); EXPECT_OK(ZipFile::OpenZipFile(&resolver, std::move(file), zip)); - const URN type(AFF4_IMAGE_TYPE); - std::unordered_set images = resolver.Query(URN(AFF4_TYPE), &type); + std::unordered_set images = resolver.Query(URN(AFF4_TYPE), URN(AFF4_IMAGE_TYPE)); ASSERT_EQ(1, images.size()); for(URN u : images){ ASSERT_EQ("aff4://8fcced2b-989f-4f51-bfa2-38d4a4d818fe", @@ -80,8 +78,7 @@ TEST_F(AFF4ImageRDFQuery, Sample3URN) { EXPECT_OK(NewFileBackedObject(&resolver, filename, "read", file)); EXPECT_OK(ZipFile::OpenZipFile(&resolver, std::move(file), zip)); - const URN type(AFF4_IMAGE_TYPE); - std::unordered_set images = resolver.Query(URN(AFF4_TYPE), &type); + std::unordered_set images = resolver.Query(URN(AFF4_TYPE), URN(AFF4_IMAGE_TYPE)); ASSERT_EQ(1, images.size()); for(URN u : images){ ASSERT_EQ("aff4://3a873665-7bf6-47b5-a12a-d6632a58ddf9", diff --git a/tools/pmem/linux_pmem.cc b/tools/pmem/linux_pmem.cc index 60a4c5d..0631271 100644 --- a/tools/pmem/linux_pmem.cc +++ b/tools/pmem/linux_pmem.cc @@ -188,7 +188,7 @@ AFF4Status LinuxPmemImager::ImagePhysicalMemory() { URN map_urn = output_volume->urn.Append("proc/kcore"); // This is a physical memory image. - resolver.Set(map_urn, AFF4_CATEGORY, new URN(AFF4_MEMORY_PHYSICAL)); + resolver.Set(map_urn, AFF4_CATEGORY, URN(AFF4_MEMORY_PHYSICAL)); if (format == "map") { RETURN_IF_ERROR(WriteMapObject_(map_urn)); @@ -198,7 +198,7 @@ AFF4Status LinuxPmemImager::ImagePhysicalMemory() { RETURN_IF_ERROR(WriteElfFormat_(map_urn)); } - resolver.Set(map_urn, AFF4_TYPE, new URN(AFF4_IMAGE_TYPE), + resolver.Set(map_urn, AFF4_TYPE, URN(AFF4_IMAGE_TYPE), /* replace = */ false); actions_run.insert("memory"); diff --git a/tools/pmem/osxpmem.cc b/tools/pmem/osxpmem.cc index 1ed9d81..a9cc956 100644 --- a/tools/pmem/osxpmem.cc +++ b/tools/pmem/osxpmem.cc @@ -141,8 +141,8 @@ AFF4Status OSXPmemImager::ImagePhysicalMemory() { output_volume_backing_urn = URN("builtin://stdout"); } - resolver.Set(output_volume_backing_urn, AFF4_STREAM_WRITE_MODE, - new XSDString("truncate")); + resolver.Set(output_volume_backing_urn, AFF4_STREAM_WRITE_MODE, + XSDString("truncate")); if (format == "elf") { return WriteElfFormat_(output_volume_backing_urn, output_volume_backing_urn); @@ -162,7 +162,7 @@ AFF4Status OSXPmemImager::ImagePhysicalMemory() { output_volume_urn); // This is a physical memory image. - resolver.Set(map_urn, AFF4_CATEGORY, new URN(AFF4_MEMORY_PHYSICAL)); + resolver.Set(map_urn, AFF4_CATEGORY, URN(AFF4_MEMORY_PHYSICAL)); if (format == "map") { res = WriteMapObject_(map_urn, output_volume_urn); diff --git a/tools/pmem/osxpmem.h b/tools/pmem/osxpmem.h index 7c709e0..b4caf7f 100644 --- a/tools/pmem/osxpmem.h +++ b/tools/pmem/osxpmem.h @@ -98,21 +98,21 @@ class OSXPmemImager: public PmemImager { void fix_path(const std::string& path, mode_t mode); virtual AFF4Status RegisterArgs() { - AddArg(new TCLAP::SwitchArg( - "L", "load-driver", "Load the driver and exit", false)); + AddArg( + "L", "load-driver", "Load the driver and exit", false); - AddArg(new TCLAP::SwitchArg( - "U", "unload-driver", "Unload the driver and exit", false)); + AddArg( + "U", "unload-driver", "Unload the driver and exit", false); - AddArg(new TCLAP::ValueArg( + AddArg>( "", "driver", "Path to driver to load. " "This is usually set to the driver included in the package.", - false, "MacPmem.kext", "Path to driver.")); + false, "MacPmem.kext", "Path to driver."); - AddArg(new TCLAP::ValueArg( + AddArg>( "", "device", "Path to device to image. " "Note the device name depends on the specific driver.", - false, "pmem", "Path to device.")); + false, "pmem", "Path to device."); return PmemImager::RegisterArgs(); } diff --git a/tools/pmem/pmem.h b/tools/pmem/pmem.h index 2b91d3b..d869408 100644 --- a/tools/pmem/pmem.h +++ b/tools/pmem/pmem.h @@ -74,35 +74,35 @@ class PmemImager: public BasicImager { virtual AFF4Status Initialize(); virtual AFF4Status RegisterArgs() { - AddArg(new TCLAP::ValueArg( + AddArg>( "", "format", "Specify the output format of memory streams:\n" " map: An AFF4Map object (Supports compression and sparse).\n" " elf: An ELF stream. (Supports sparse image).\n" " raw: A raw padded stream. (Padded with no compression).\n" "If this option is used together with the --export option it " "specifies the output format of the exported stream.", - false, "map", "map, elf, raw")); + false, "map", "map, elf, raw"); - AddArg(new TCLAP::ValueArg( + AddArg>( "", "volume_format", "Specify the output format type:\n" " aff4: The output will be an AFF4 volume\n" " raw: The output will be a raw file. NOTE: Only one " "stream can be written in this case.\n", - false, "aff4", "aff4, raw")); + false, "aff4", "aff4, raw"); - AddArg(new TCLAP::SwitchArg( + AddArg( "m", "acquire-memory", "Normally pmem will only acquire memory if " "the user has not asked for something else (like acquiring files, " "exporting etc). This option forces memory to be acquired. It is only " "required when the program is invoked with the --input, --export or " - "other actionable flags.\n", false)); + "other actionable flags.\n", false); - AddArg(new TCLAP::MultiArgToNextFlag( + AddArg( "p", "pagefile", "Also capture the pagefile. Note that you must " "provide this option rather than e.g. '--input c:\\pagefile.sys' " "because we cannot normally read the pagefile directly. This " "option will use the sleuthkit to read the pagefile.", - false, "/path/to/pagefile")); + false, "/path/to/pagefile"); return BasicImager::RegisterArgs(); } diff --git a/tools/pmem/pmem_imager.cc b/tools/pmem/pmem_imager.cc index 64dc623..fddf264 100644 --- a/tools/pmem/pmem_imager.cc +++ b/tools/pmem/pmem_imager.cc @@ -127,7 +127,7 @@ AFF4Status PmemImager::handle_compression() { AFF4Status PmemImager::WriteElfFormat_(const URN &output_urn) { resolver.logger->info("Will write in ELF format."); - AFF4Flusher header(new StringIO(&resolver)); + auto header = make_flusher(&resolver); // Create a temporary map for WriteStream() API. AFF4Map memory_layout(&resolver); @@ -245,8 +245,8 @@ AFF4Status PmemImager::WriteMapObject_(const URN &map_urn) { RETURN_IF_ERROR(CreateMap_(&temp_stream, &total_length)); // Set the user's preferred compression method. - resolver.Set(map_urn.Append("data"), AFF4_IMAGE_COMPRESSION, new URN( - CompressionMethodToURN(compression))); + resolver.Set(map_urn.Append("data"), AFF4_IMAGE_COMPRESSION, + CompressionMethodToURN(compression)); // Create the map object. AFF4Volume *volume; diff --git a/tools/pmem/win_pmem.cc b/tools/pmem/win_pmem.cc index eb304cf..e1fce12 100644 --- a/tools/pmem/win_pmem.cc +++ b/tools/pmem/win_pmem.cc @@ -327,9 +327,9 @@ AFF4Status WinPmemImager::ImagePageFile() { AFF4Flusher pagefile; RETURN_IF_ERROR(GetWritableStream_(pagefile_urn, pagefile)); - resolver.Set(pagefile->urn, AFF4_CATEGORY, new URN(AFF4_MEMORY_PAGEFILE)); + resolver.Set(pagefile->urn, AFF4_CATEGORY, URN(AFF4_MEMORY_PAGEFILE)); resolver.Set(pagefile->urn, AFF4_MEMORY_PAGEFILE_NUM, - new XSDInteger(pagefile_number)); + XSDInteger(pagefile_number)); VolumeManager progress(&resolver, this); @@ -381,8 +381,8 @@ AFF4Status WinPmemImager::WriteMapObject_(const URN &map_urn) { data_stream.get(), map_stream)); // Set the user's preferred compression method on the data stream. - resolver.Set(data_stream->urn, AFF4_IMAGE_COMPRESSION, new URN( - CompressionMethodToURN(compression))); + resolver.Set(data_stream->urn, AFF4_IMAGE_COMPRESSION, + CompressionMethodToURN(compression)); // Get the ranges from the memory device. PmemMemoryInfo info; @@ -434,9 +434,8 @@ AFF4Status WinPmemImager::WriteMapObject_(const URN &map_urn) { // This will happen when windows is running in VSM mode - // some of the physical pages are not actually accessible. - AFF4Flusher unreadable( - new AFF4SymbolicStream(&resolver, AFF4_IMAGESTREAM_UNREADABLE, - "UNREADABLEDATA")); + auto unreadable = make_flusher(&resolver, + AFF4_IMAGESTREAM_UNREADABLE, "UNREADABLEDATA"); // One of the pages in the range is unreadable - repeat // the read for each page. @@ -540,7 +539,7 @@ AFF4Status WinPmemImager::ImagePhysicalMemory() { } // This is a physical memory image. - resolver.Set(map_urn, AFF4_CATEGORY, new URN(AFF4_MEMORY_PHYSICAL)); + resolver.Set(map_urn, AFF4_CATEGORY, URN(AFF4_MEMORY_PHYSICAL)); actions_run.insert("memory"); @@ -569,7 +568,7 @@ AFF4Status WinPmemImager::ExtractFile_( std::string output_file) { private_resolver.Set(output_file, AFF4_STREAM_WRITE_MODE, - new XSDString("truncate")); + XSDString("truncate")); AFF4Flusher output; RETURN_IF_ERROR(NewFileBackedObject( diff --git a/tools/pmem/win_pmem.h b/tools/pmem/win_pmem.h index 0d8bd4e..6e8e28e 100644 --- a/tools/pmem/win_pmem.h +++ b/tools/pmem/win_pmem.h @@ -145,25 +145,25 @@ class WinPmemImager: public PmemImager { AFF4Status UninstallDriver(); AFF4Status RegisterArgs() override { - AddArg(new TCLAP::SwitchArg( - "L", "load-driver", "Load the driver and exit", false)); + AddArg( + "L", "load-driver", "Load the driver and exit", false); - AddArg(new TCLAP::SwitchArg( - "U", "unload-driver", "Unload the driver and exit", false)); + AddArg( + "U", "unload-driver", "Unload the driver and exit", false); - AddArg(new TCLAP::SwitchArg( + AddArg( "", "write-mode", "Enable write mode. You must have the " "driver compiled with write support and be on a system with " - "test signing enabled.", false)); + "test signing enabled.", false); - AddArg(new TCLAP::ValueArg( + AddArg>( "", "mode", "Select the acquisition mode. Default is PTERemapping.", - false, "", "MmMapIoSpace, PhysicalMemory, PTERemapping")); + false, "", "MmMapIoSpace, PhysicalMemory, PTERemapping"); - AddArg(new TCLAP::ValueArg( + AddArg>( "", "driver", "Use this driver instead of the included one. " "This option is rarely used.", - false, "", "Path to driver.")); + false, "", "Path to driver."); return PmemImager::RegisterArgs(); } From e752f2f7611987781293f63d9c182644c196260c Mon Sep 17 00:00:00 2001 From: Joe Sylve Date: Tue, 9 Jul 2019 00:54:32 -0500 Subject: [PATCH 2/5] Fixed issue with compiling with gcc --- aff4/attributes.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/aff4/attributes.h b/aff4/attributes.h index 512e9b4..45fd231 100644 --- a/aff4/attributes.h +++ b/aff4/attributes.h @@ -70,9 +70,10 @@ class AttributeValue { * Types must inherit from RDFValue and also need to be added * to the type enum below (in order). */ - using storage = absl::variant; + using storage = absl::variant; static_assert(validate_base::value, "storage types must all inherit from RDFValue"); From 3053c4c93e4ec23c23b5b0884a2b726a5a2e82d9 Mon Sep 17 00:00:00 2001 From: Joe Sylve Date: Mon, 19 Aug 2019 12:52:30 -0500 Subject: [PATCH 3/5] Fixed compile error from merge --- aff4/aff4_imager_utils.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aff4/aff4_imager_utils.h b/aff4/aff4_imager_utils.h index 03175c4..6228bdf 100644 --- a/aff4/aff4_imager_utils.h +++ b/aff4/aff4_imager_utils.h @@ -238,7 +238,7 @@ class BasicImager { AddArg>( "c", "compression", "Type of compression to use (default deflate).", - false, "", "deflate, zlib, snappy, lz4, none")); + false, "", "deflate, zlib, snappy, lz4, none"); AddArg>( "", "threads", "Total number of threads to use.", From 0dcb287e3196fbe8976db36fac9b4699a8097a77 Mon Sep 17 00:00:00 2001 From: Joe Sylve Date: Mon, 19 Aug 2019 14:25:14 -0500 Subject: [PATCH 4/5] Support querying all properties as strings from the C API --- aff4/libaff4-c.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aff4/libaff4-c.cc b/aff4/libaff4-c.cc index 39359ab..6b61266 100644 --- a/aff4/libaff4-c.cc +++ b/aff4/libaff4-c.cc @@ -361,13 +361,13 @@ int AFF4_get_string_property(AFF4_Handle* handle, const char * property, char** get_log_handler().use(msg); - aff4::XSDString value; + aff4::AttributeValue value; if (handle->resolver.Get(handle->urn, aff4::URN(property), value) != aff4::STATUS_OK) { errno = ENOENT; return -1; } - const std::string & res = value.value; + const std::string res = value->SerializeToString(); *result = (char *) malloc(res.length() + 1); if (*result == nullptr) { From 41f768b61771938c68326b12cb95a2ee40753c4b Mon Sep 17 00:00:00 2001 From: Joe Sylve Date: Thu, 22 Aug 2019 14:48:49 -0500 Subject: [PATCH 5/5] Add ability to open select streams and to fetch multi-value properties from the C API --- aff4/libaff4-c.cc | 150 ++++++++++++++++++++++++++++++++++++++-------- aff4/libaff4-c.h | 46 ++++++++++++++ 2 files changed, 171 insertions(+), 25 deletions(-) diff --git a/aff4/libaff4-c.cc b/aff4/libaff4-c.cc index 6b61266..ebb6a55 100644 --- a/aff4/libaff4-c.cc +++ b/aff4/libaff4-c.cc @@ -94,20 +94,17 @@ static std::shared_ptr get_c_api_logger() { } struct AFF4_Handle { - aff4::MemoryDataStore resolver; - aff4::URN urn; - aff4::VolumeGroup volumes; - aff4::AFF4Flusher stream; - std::string filename; - - AFF4_Handle(const std::string & filename): - resolver(aff4::DataStoreOptions(get_c_api_logger(), 1)), - volumes(&resolver), - filename(filename) - {} + aff4::MemoryDataStore resolver{aff4::DataStoreOptions(get_c_api_logger(), 1)}; + aff4::VolumeGroup volumes{&resolver}; + aff4::AFF4Flusher stream{}; + std::string filename{}; + aff4::URN urn{}; + bool autoload{}; + + AFF4_Handle(const std::string & filename) : filename(filename) {} protected: - bool open() { + bool open(const aff4::URN& stream_urn = {}) { aff4::AFF4Flusher file; if (aff4::STATUS_OK != aff4::NewFileBackedObject( &resolver, filename, "read", file)) { @@ -122,8 +119,24 @@ struct AFF4_Handle { volumes.AddVolume(std::move(zip)); - // Attempt AFF4 Standard, and if not, fallback to AFF4 Evimetry Legacy format. + if (stream_urn.value.empty()) { + if (!load_default_urn()) { + return false; + } + } else { + urn = stream_urn; + } + if (aff4::STATUS_OK != volumes.GetStream(urn, stream)) { + return false; + } + + return true; + } + + // Finds the first aff4:Image in the volume and selects it for loading + bool load_default_urn() { + // Attempt AFF4 Standard, and if not, fallback to AFF4 Evimetry Legacy format. auto images = resolver.Query(aff4::AFF4_TYPE, aff4::URN(aff4::AFF4_IMAGE_TYPE)); if (images.empty()) { @@ -137,9 +150,7 @@ struct AFF4_Handle { // For determinism, get the "first" sorted urn in the set urn = *std::min_element(images.begin(), images.end()); - if (aff4::STATUS_OK != volumes.GetStream(urn, stream)) { - return false; - } + autoload = true; return true; } @@ -152,12 +163,27 @@ class HandlePool { public: - AFF4_Handle * get(const std::string & filename) { + AFF4_Handle * get(const std::string & filename, + const aff4::URN & urn = {}) { std::lock_guard lock{pool_lock}; const auto it = std::find_if(pool.begin(), pool.end(), - [&filename](const std::unique_ptr & el) { - return (el != nullptr && el->filename == filename); + [&](const std::unique_ptr & el) { + if (el == nullptr || el->filename != filename) { + return false; + } + + if (urn.value.empty()) { + if (el->autoload == false) { + return false; + } + } else { + if (urn != el->urn) { + return false; + } + } + + return true; }); if (it != pool.end()) { @@ -166,7 +192,7 @@ class HandlePool { auto h = absl::make_unique(filename); - if (h->open()) { + if (h->open(urn)) { return h.release(); } @@ -274,6 +300,20 @@ AFF4_Handle* AFF4_open(const char* filename, AFF4_Message** msg) { return h; } +AFF4_Handle* AFF4_open_stream(const char* filename, + const char* urn, + AFF4_Message** msg) { + get_log_handler().use(msg); + + AFF4_Handle * h = handle_pool().get(filename, urn); + + if (h == nullptr) { + errno = ENOENT; + } + + return h; +} + uint64_t AFF4_object_size(AFF4_Handle* handle, AFF4_Message** msg) { get_log_handler().use(msg); @@ -357,7 +397,9 @@ int AFF4_get_string_property(AFF4_Handle* handle, const char * property, char** return -1; } - *result = nullptr; + auto & resultPtr = *result; + + resultPtr = nullptr; get_log_handler().use(msg); @@ -369,14 +411,14 @@ int AFF4_get_string_property(AFF4_Handle* handle, const char * property, char** const std::string res = value->SerializeToString(); - *result = (char *) malloc(res.length() + 1); - if (*result == nullptr) { + resultPtr = (char *) malloc(res.length() + 1); + if (resultPtr == nullptr) { errno = ENOMEM; return -1; } - res.copy(*result, res.length()); - *result[res.length()] = 0; // null terminate string + res.copy(resultPtr, res.length()); + resultPtr[res.length()] = 0; // null terminate string return 0; } @@ -411,4 +453,62 @@ int AFF4_get_binary_property(AFF4_Handle* handle, const char * property, AFF4_Bi return 0; } +void AFF4_free_properties(AFF4_Property * properties) { + if (properties == nullptr) { + return; + } + + AFF4_Property * property = properties; + do { + if (property->value) { + delete [] property->value; + } + + property = property->next; + } while(property != nullptr); + + delete [] properties; +} + +int AFF4_get_properties(AFF4_Handle* handle, const char * property, AFF4_Property** result, AFF4_Message** msg) { + if (!handle || !property || !result) { + errno = EINVAL; + return -1; + } + + get_log_handler().use(msg); + + auto & resultPtr = *result; + + resultPtr = nullptr; + + std::vector values; + if (handle->resolver.Get(handle->urn, aff4::URN(property), values) != aff4::STATUS_OK) { + errno = ENOENT; + return -1; + } + + resultPtr = new AFF4_Property[values.size()](); + + for(size_t i = 0; i < values.size(); i++) { + auto & property = resultPtr[i]; + const auto & attribute = values[i]; + + if (i != values.size() - 1) { + property.next = &resultPtr[i+1]; + } + + property.type = static_cast(attribute.Type()); + + std::string value = attribute->SerializeToString(); + + property.value = new char[value.length() + 1](); + + value.copy(property.value, value.length()); + + } + + return 0; +} + } diff --git a/aff4/libaff4-c.h b/aff4/libaff4-c.h index cbb18bc..96ca44a 100644 --- a/aff4/libaff4-c.h +++ b/aff4/libaff4-c.h @@ -95,6 +95,17 @@ typedef struct AFF4_Handle AFF4_Handle; */ AFF4_Handle* AFF4_open(const char* filename, AFF4_Message** msg); +/** + * Open the given filename, and access the first aff4:Image in the container. + * @param filename The filename to open. + * @param urn The URN of the stream to open. + * @param msg A pointer to log messages. + * @return Object handle, or NULL on error. See errno + */ +AFF4_Handle* AFF4_open_stream(const char* filename, + const char* urn, + AFF4_Message** msg); + /** * Get the size of the AFF4 Object that was opened. */ @@ -163,6 +174,41 @@ typedef struct { */ int AFF4_get_binary_property(AFF4_Handle* handle, const char * property, AFF4_Binary_Result* result, AFF4_Message** msg); +typedef enum { + INVALID, + RDFBytes, + XSDString, + MD5Hash, + SHA1Hash, + SHA256Hash, + SHA512Hash, + Blake2BHash, + XSDInteger, + XSDBoolean, + URN +} AFF4_Property_Type; + +typedef struct _AFF4_Property { + AFF4_Property_Type type; + char * value; + struct _AFF4_Property * next; +} AFF4_Property; + +/** + * Free AFF4_Property memory + */ +void AFF4_free_properties(AFF4_Property * properties); + +/** + * @param handle The Object handle. + * @param property The property key + * @param result Pointer to store the results (must be freed by AFF4_free_properties) + * @param msg A pointer to log messages. + * @return 0 on success or non-zero on error + */ +int AFF4_get_properties(AFF4_Handle* handle, const char * property, AFF4_Property** result, AFF4_Message** msg); + + #ifdef __cplusplus } #endif