-
Notifications
You must be signed in to change notification settings - Fork 65
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This is still experimental.
- Loading branch information
Showing
16 changed files
with
567 additions
and
61 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,3 +24,4 @@ Roxygen: list(markdown = TRUE) | |
| Suggests: | ||
| spelling | ||
| Language: en-GB | ||
| Encoding: UTF-8 | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| #' GridFS API | ||
| #' | ||
| #' Connect to a GridFS database. | ||
| #' | ||
| #' @inheritParams mongo | ||
| #' @export | ||
| #' @param prefix string to prefix the collection name | ||
| #' @examples # New GridFS | ||
| #' fs <- gridfs(url = "mongodb+srv://readwrite:[email protected]/test") | ||
| #' input <- R.home('doc/NEWS.pdf') | ||
| #' fs$upload(input) | ||
| #' fs$download('NEWS.pdf', 'output.pdf') | ||
| #' hashes <- tools::md5sum(c(input, 'output.pdf')) | ||
| #' unlink('output.pdf') | ||
| #' stopifnot(hashes[[1]] == hashes[[1]]) | ||
| #' | ||
| #' # Insert Binary Data | ||
| #' fs$write('iris3', serialize(datasets::iris3, NULL)) | ||
| #' out <- unserialize(fs$read('iris3')) | ||
| #' stopifnot(all.equal(out, datasets::iris3)) | ||
| #' | ||
| #' # Show what we have | ||
| #' fs$list() | ||
| #' fs$drop() | ||
| gridfs <- function(db = "test", url = "mongodb://localhost", prefix = "fs", options = ssl_options()){ | ||
| client <- do.call(mongo_client_new, c(list(uri = url), options)) | ||
|
|
||
| # Get a database | ||
| if(missing(db) || is.null(db)){ | ||
| url_db <- mongo_get_default_database(client) | ||
| if(length(url_db) && nchar(url_db)) | ||
| db <- url_db | ||
| } | ||
|
|
||
| fs <- mongo_gridfs_new(client, prefix, db) | ||
| orig <- list( | ||
| prefix = prefix, | ||
| db = db, | ||
| url = url, | ||
| options = options | ||
| ) | ||
| if(length(options$pem_file) && file.exists(options$pem_file)) | ||
| attr(orig, "pemdata") <- readLines(options$pem_file) | ||
| fs_object(fs, client, orig) | ||
| } | ||
|
|
||
| fs_object <- function(fs, client, orig){ | ||
| self <- local({ | ||
| drop <- function(){ | ||
| mongo_gridfs_drop(fs) | ||
| } | ||
| list <- function(filter = '{}', options = '{}'){ | ||
| mongo_gridfs_list(fs, filter, options) | ||
| } | ||
| upload <- function(path, name = basename(path)){ | ||
| mongo_gridfs_upload(fs, name, path) | ||
| } | ||
| download <- function(name, path = name){ | ||
| mongo_gridfs_download(fs, name, path) | ||
| } | ||
| read <- function(name){ | ||
| mongo_gridfs_read(fs, name) | ||
| } | ||
| write <- function(name, data){ | ||
| mongo_gridfs_write(fs, name, data) | ||
| } | ||
| remove <- function(name){ | ||
| mongo_gridfs_remove(fs, name) | ||
| } | ||
| environment() | ||
| }) | ||
| lockEnvironment(self, TRUE) | ||
| structure(self, class=c("gridfs", "jeroen", class(self))) | ||
| } | ||
|
|
||
| #' @useDynLib mongolite R_mongo_gridfs_new | ||
| mongo_gridfs_new <- function(client, prefix, db){ | ||
| .Call(R_mongo_gridfs_new, client, prefix, db) | ||
| } | ||
|
|
||
| #' @useDynLib mongolite R_mongo_gridfs_drop | ||
| mongo_gridfs_drop <- function(fs){ | ||
| .Call(R_mongo_gridfs_drop, fs) | ||
| } | ||
|
|
||
| #' @useDynLib mongolite R_mongo_gridfs_list | ||
| mongo_gridfs_list <- function(fs, filter, opts){ | ||
| out <- .Call(R_mongo_gridfs_list, fs, bson_or_json(filter), bson_or_json(opts)) | ||
| out <- lapply(out, unlist, recursive = TRUE) | ||
| names(out) <- c('id', 'name', 'size', 'date') | ||
| out$date <- structure(out$date / 1000, class = c("POSIXct", "POSIXt")) | ||
| data.frame(out, stringsAsFactors = FALSE) | ||
| } | ||
|
|
||
| #' @useDynLib mongolite R_mongo_gridfs_upload | ||
| mongo_gridfs_upload <- function(fs, name, path){ | ||
| stopifnot(is.character(name)) | ||
| path <- normalizePath(path, mustWork = TRUE) | ||
| stopifnot(length(name) == length(path)) | ||
| id <- rep(NA, length(name)) | ||
| for(i in seq_along(name)){ | ||
| out <- .Call(R_mongo_gridfs_upload, fs, name[i], path[i]) | ||
| id[i] = out$id | ||
| } | ||
| structure(id, names = name) | ||
| } | ||
|
|
||
| #' @useDynLib mongolite R_mongo_gridfs_download | ||
| mongo_gridfs_download <- function(fs, name, path){ | ||
| stopifnot(is.character(name)) | ||
| path <- normalizePath(path, mustWork = FALSE) | ||
| stopifnot(length(name) == length(path)) | ||
| out <- rep(NA, length(name)) | ||
| for(i in seq_along(name)){ | ||
| out <- .Call(R_mongo_gridfs_download, fs, name[i], path[i]) | ||
| } | ||
| structure(out, names = name) | ||
| } | ||
|
|
||
| #' @useDynLib mongolite R_mongo_gridfs_write | ||
| mongo_gridfs_write <- function(fs, name, data){ | ||
| stopifnot(is.raw(data)) | ||
| stopifnot(is.character(name)) | ||
| .Call(R_mongo_gridfs_write, fs, name, data) | ||
| } | ||
|
|
||
| #' @useDynLib mongolite R_mongo_gridfs_read | ||
| mongo_gridfs_read <- function(fs, name){ | ||
| .Call(R_mongo_gridfs_read, fs, name) | ||
| } | ||
|
|
||
| #' @useDynLib mongolite R_mongo_gridfs_remove | ||
| mongo_gridfs_remove <- function(fs, name){ | ||
| vapply(name, function(x){ | ||
| .Call(R_mongo_gridfs_remove, fs, x)$id | ||
| }, character(1)) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,109 @@ | ||
| --- | ||
| title: "Proposal: GridFS Design in mongolite" | ||
| output: | ||
| word_document: default | ||
| html_document: | ||
| df_print: paged | ||
| --- | ||
|
|
||
| ```{r setup, include=FALSE} | ||
| knitr::opts_chunk$set(echo = TRUE) | ||
| ``` | ||
|
|
||
| GridFS API in mongolite will be a new top level object class, consistent with the current API to instantiate regular mongodb collection objects. | ||
|
|
||
| ```r | ||
| fs <- mongolite::gridfs(db = "test", url = "mongodb://localhost") | ||
| ``` | ||
|
|
||
| The initial API will focus on basic read/write/delete operations. | ||
|
|
||
| ## Error Handling | ||
|
|
||
| All methods will automatically translate `mongoc` exceptions into R errors. | ||
|
|
||
| ## Listing Files | ||
|
|
||
| Returns a data frame with files and fixed meta data (size, date, content-type, etc). | ||
|
|
||
| ```r | ||
| list <- fs$list(filter = '{}', options = '{}') | ||
| ``` | ||
|
|
||
| __*References*__: | ||
|
|
||
| - [mongoc_gridfs_find_with_opts](http://mongoc.org/libmongoc/current/mongoc_gridfs_t.html) for listing | ||
| - [mongoc_gridfs_file_t](http://mongoc.org/libmongoc/current/mongoc_gridfs_file_t.html) for reading file properties | ||
|
|
||
| ## Reading Files | ||
|
|
||
| A file can be read either into a buffer, or streamed to a file or connection. The default behavior is to read the entire file and return the data in a raw data vector: | ||
|
|
||
| ```r | ||
| buf <- fs$read(name = "myfile.bin") | ||
| ``` | ||
|
|
||
| Alternatively the user can supply an R connection object that we can use to stream data to e.g. a file or network socket. | ||
|
|
||
| ```r | ||
| fs$read(name = "myfile.bin", con = connection) | ||
| ``` | ||
|
|
||
| The latter will be a memory efficient way to incrementally read from the GridFS and write out the data. It is similar to the `export()` method for regular mongo collection objects. | ||
|
|
||
|
|
||
| __*References*__: | ||
|
|
||
| - [mongoc_gridfs_find_one_by_filename](http://mongoc.org/libmongoc/current/mongoc_gridfs_find_one_by_filename.html) to lookup the file | ||
| - [mongoc_stream_gridfs_new](http://mongoc.org/libmongoc/current/mongoc_stream_gridfs_new.html) to create a stream reader | ||
| - [mongoc_stream_t](http://mongoc.org/libmongoc/current/mongoc_stream_t.html) methods for reading the stream | ||
| - [What exactly is a connection in R](https://stackoverflow.com/questions/30445875/what-exactly-is-a-connection-in-r) | ||
|
|
||
|
|
||
| ## Writing Files | ||
|
|
||
| Analogous to reading, a write operation can either write a raw data vector from memory or stream data from a local file or connection object. | ||
|
|
||
| ```r | ||
| fs$write(name = "myfile.bin", data = buffer) | ||
| ``` | ||
|
|
||
| When the `data` argument is an R connection object, it will incrementally read from the connection and upload to GridFS. | ||
|
|
||
| ```r | ||
| fs$write(name = "myfile.bin", data = connection) | ||
| ``` | ||
|
|
||
|
|
||
| __*References*__: | ||
|
|
||
| - [mongoc_gridfs_create_file_from_stream](http://mongoc.org/libmongoc/current/mongoc_gridfs_create_file_from_stream.html) create a new file using a stream | ||
| - [mongoc_stream_write](http://mongoc.org/libmongoc/current/mongoc_stream_write.html) write to the stream | ||
| - [What exactly is a connection in R](https://stackoverflow.com/questions/30445875/what-exactly-is-a-connection-in-r) | ||
|
|
||
| ## Removing Files | ||
|
|
||
| Removes a single file from the GridFS collection: | ||
|
|
||
| ```r | ||
| fs$remove(name = "myfile.bin") | ||
| ``` | ||
|
|
||
| Here the `name` argument can be vectorized in standard R fashion such that multiple files can be removed with a single call. | ||
|
|
||
| __*References*__: | ||
|
|
||
| - [mongoc_gridfs_remove_by_filename](http://mongoc.org/libmongoc/current/mongoc_gridfs_remove_by_filename.html) to delete the file | ||
|
|
||
| ## Drop GridFS | ||
|
|
||
| Requests that an entire GridFS be dropped, including all files associated with it. | ||
|
|
||
| ```r | ||
| fs$drop() | ||
| ``` | ||
|
|
||
| __*References*__: | ||
|
|
||
| - [mongoc_gridfs_drop](http://mongoc.org/libmongoc/current/mongoc_gridfs_drop.html) | ||
|
|
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.