From 28b922c09a3b8758c3d814d641d963040de3b0dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Revol?= Date: Mon, 13 Jan 2020 17:40:16 +0100 Subject: [PATCH] WIP: ddrescue mapfile filter This allows to overlay bad sectors according to the mapfile generated by ddrescue, to then see where sectors are used using fsck and trying to copy files around. --- configure.ac | 2 + filters/ddrescue/Makefile.am | 75 ++++++ filters/ddrescue/ddrescue.c | 218 +++++++++++++++ filters/ddrescue/nbdkit-ddrescue-filter.pod | 282 ++++++++++++++++++++ 4 files changed, 577 insertions(+) create mode 100644 filters/ddrescue/Makefile.am create mode 100644 filters/ddrescue/ddrescue.c create mode 100644 filters/ddrescue/nbdkit-ddrescue-filter.pod diff --git a/configure.ac b/configure.ac index 5f15b6419..d87529dfd 100644 --- a/configure.ac +++ b/configure.ac @@ -895,6 +895,7 @@ filters="\ cache \ cacheextents \ cow \ + ddrescue \ delay \ error \ fua \ @@ -981,6 +982,7 @@ AC_CONFIG_FILES([Makefile filters/cache/Makefile filters/cacheextents/Makefile filters/cow/Makefile + filters/ddrescue/Makefile filters/delay/Makefile filters/error/Makefile filters/fua/Makefile diff --git a/filters/ddrescue/Makefile.am b/filters/ddrescue/Makefile.am new file mode 100644 index 000000000..2498074c2 --- /dev/null +++ b/filters/ddrescue/Makefile.am @@ -0,0 +1,75 @@ +# nbdkit +# Copyright (C) 2018 Red Hat Inc. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Red Hat nor the names of its contributors may be +# used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. + +include $(top_srcdir)/common-rules.mk + +EXTRA_DIST = \ + nbdkit-ddrescue-filter.pod \ + $(NULL) + +filter_LTLIBRARIES = nbdkit-ddrescue-filter.la + +nbdkit_ddrescue_filter_la_SOURCES = \ + ddrescue.c \ + $(top_srcdir)/include/nbdkit-filter.h \ + $(NULL) + +nbdkit_ddrescue_filter_la_CPPFLAGS = \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/common/include \ + -I$(top_srcdir)/common/sparse \ + -I$(top_srcdir)/common/utils \ + $(NULL) +nbdkit_ddrescue_filter_la_CFLAGS = \ + $(WARNINGS_CFLAGS) \ + $(GNUTLS_CFLAGS) \ + $(NULL) +nbdkit_ddrescue_filter_la_LDFLAGS = \ + -module -avoid-version -shared \ + -Wl,--version-script=$(top_srcdir)/filters/filters.syms \ + $(NULL) +nbdkit_ddrescue_filter_la_LIBADD = \ + $(top_builddir)/common/sparse/libsparse.la \ + $(top_builddir)/common/utils/libutils.la \ + $(GNUTLS_LIBS) \ + $(NULL) + +if HAVE_POD + +man_MANS = nbdkit-ddrescue-filter.1 +CLEANFILES += $(man_MANS) + +nbdkit-ddrescue-filter.1: nbdkit-ddrescue-filter.pod + $(PODWRAPPER) --section=1 --man $@ \ + --html $(top_builddir)/html/$@.html \ + $< + +endif HAVE_POD diff --git a/filters/ddrescue/ddrescue.c b/filters/ddrescue/ddrescue.c new file mode 100644 index 000000000..5a125e8a2 --- /dev/null +++ b/filters/ddrescue/ddrescue.c @@ -0,0 +1,218 @@ +/* nbdkit + * Copyright (C) 2018-2019 Red Hat Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of Red Hat nor the names of its contributors may be + * used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY RED HAT AND CONTRIBUTORS ''AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL RED HAT OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#include +#include +#include +#include +#include + +#include + +#include + +#include "cleanup.h" + +struct range { + int64_t start; + int64_t end; + int64_t size; + char status; +}; + +struct mapfile { + int ranges_count; + struct range *ranges; +}; + +static struct mapfile map = { 0, NULL }; + +static int +parse_mapfile (const char *filename) +{ + FILE *fp = NULL; + CLEANUP_FREE char *line = NULL; + size_t linelen = 0; + ssize_t len; + int ret = -1; + int status_seen = 0; + + fp = fopen (filename, "r"); + if (!fp) { + nbdkit_error ("%s: ddrescue: fopen: %m", filename); + goto out; + } + + while ((len = getline (&line, &linelen, fp)) != -1) { + const char *delim = " \t"; + char *sp, *p; + int64_t offset, length; + char status; + + if (len > 0 && line[len-1] == '\n') { + line[len-1] = '\0'; + len--; + } + + if (len > 0 && line[0] == '#') + continue; + + if (len > 0 && !status_seen) { + /* status line, ignore it for now */ + status_seen = 1; + nbdkit_debug ("%s: skipping status line: '%s'", filename, line); + continue; + } + + if (sscanf (line, "%" SCNi64 "\t%" SCNi64 "\t%c", &offset, &length, &status) == 3) { + if (offset < 0) { + nbdkit_error ("block offset must not be negative"); + return -1; + } + if (length < 0) { + nbdkit_error ("block length must not be negative"); + return -1; + } + if (status == '+') { + int i = map.ranges_count++; + map.ranges = realloc(map.ranges, map.ranges_count * sizeof(struct range)); + if (map.ranges == NULL) { + nbdkit_error ("%s: ddrescue: realloc: %m", filename); + goto out; + } + map.ranges[i].start = offset; + map.ranges[i].end = offset + length - 1; + map.ranges[i].size = length; + map.ranges[i].status = status; + } + + nbdkit_debug ("%s: range: 0x%" PRIx64 " 0x%" PRIx64 " '%c'", filename, offset, length, status); + } + } + + ret = 0; + + out: + if (fp) + fclose (fp); + return ret; +} + + +static void +ddrescue_load (void) +{ +} + +/* On unload, free the sparse array. */ +static void +ddrescue_unload (void) +{ + free (map.ranges); + map.ranges = NULL; + map.ranges_count = 0; +} + +static int +ddrescue_config (nbdkit_next_config *next, void *nxdata, + const char *key, const char *value) +{ + if (strcmp (key, "ddrescue-mapfile") == 0) { + if (parse_mapfile (value) == -1) + return -1; + return 0; + } + + else + return next (nxdata, key, value); +} + +#define ddrescue_config_help \ + "ddrescue-mapfile=... Specify ddrescue mapfile to use" + +/* We need this because otherwise the layer below can_write is called + * and that might return true (eg. if the plugin has a pwrite method + * at all), resulting in writes being passed through to the layer + * below. This is possibly a bug in nbdkit. + */ +static int +ddrescue_can_write (struct nbdkit_next_ops *next_ops, void *nxdata, + void *handle) +{ + return 0; +} + +static int +ddrescue_can_cache (struct nbdkit_next_ops *next_ops, void *nxdata, + void *handle) +{ + return 0; +} + +/* Read data. */ +static int +ddrescue_pread (struct nbdkit_next_ops *next_ops, void *nxdata, + void *handle, void *buf, uint32_t count, uint64_t offset, + uint32_t flags, int *err) +{ + int i; + + for (i = 0; i < map.ranges_count; i++) { + if (map.ranges[i].status != '+') + continue; + if (offset >= map.ranges[i].start && offset <= map.ranges[i].end) { + if (offset + count - 1 <= map.ranges[i].end) { + /* entirely contained within this range */ + return next_ops->pread (nxdata, buf, count, offset, flags, err); + } + } + } + /* read was not fully covered */ + *err = EIO; + return -1; +} + +static struct nbdkit_filter filter = { + .name = "ddrescue", + .longname = "nbdkit ddrescue mapfile filter", + .load = ddrescue_load, + .unload = ddrescue_unload, + .config = ddrescue_config, + .config_help = ddrescue_config_help, + .can_write = ddrescue_can_write, + .can_cache = ddrescue_can_cache, + .pread = ddrescue_pread, +}; + +NBDKIT_REGISTER_FILTER(filter) diff --git a/filters/ddrescue/nbdkit-ddrescue-filter.pod b/filters/ddrescue/nbdkit-ddrescue-filter.pod new file mode 100644 index 000000000..c57f9de26 --- /dev/null +++ b/filters/ddrescue/nbdkit-ddrescue-filter.pod @@ -0,0 +1,282 @@ +=head1 NAME + +nbdkit-ddrescue-plugin - nbdkit plugin for serving from ddrescue dump + +=head1 SYNOPSIS + + nbdkit ddrescue ddrescue="0 1 2 3 @0x1fe 0x55 0xaa" [size=size] + + nbdkit ddrescue base64="aGVsbG8gbmJka2l0IHVzZXI=" [size=size] + + nbdkit ddrescue raw="binary_ddrescue" [size=size] + +=head1 DESCRIPTION + +C is a plugin for L which serves a +small amount of data specified directly on the command line. The +plugin gets its name from the C URI scheme used by web +browsers. This is mainly useful for testing NBD clients. + +You can serve data read-only using the I<-r> flag, or read-write. Any +writes are thrown away when nbdkit exits. + +Most operating systems have command line size limits which are quite a +lot smaller than any desirable disk image, so specifying a large, +fully populated disk image on the command line would not be possible. +However you can specify a small amount of data at the beginning of the +image, possibly followed by zeroes (using the C parameter to pad +the image to the full size), or use the C parameter creatively +to make mostly sparse disk images. + +The C parameter can specify any virtual size up to the maximum +supported by nbdkit (S<2⁶³-1 bytes>). + +=head1 EXAMPLES + +=over 4 + +=item Create a 1 MB empty disk: + + nbdkit ddrescue raw= size=1M + +(This is a contrived example, it is better to use +L for real applications.) + +=item Create a 1 MB disk with some nonsense data at the beginning: + + nbdkit ddrescue base64=MTIz size=1M + +The above command serves the bytes C<0x31 0x32 0x33> (which is the +base64 decoding of C), followed by S<1M - 3 bytes> of zeroes. + +=item Create a 1 MB disk with one empty MBR-formatted partition: + + nbdkit ddrescue data=" + @0x1b8 178 190 207 221 0 0 0 0 2 0 131 32 32 0 1 0 0 0 255 7 + @0x1fe 85 170 + " size=1048576 + +This example was created by running: + + $ rm -f disk + $ truncate -s 1M disk + $ echo start=1 | sfdisk disk + Device Boot Start End Sectors Size Id Type + disk1 1 2047 2047 1023.5K 83 Linux + $ ./disk2data.pl disk + +The C script is provided in the nbdkit sources +(L). + +=item Create a disk with a partition from another file: + + nbdkit ddrescue data=" + @0x1b8 178 190 207 221 0 0 0 0 2 0 131 32 32 0 1 0 0 0 255 7 + @0x1fe 85 170 + @0x200 FILE> syntax is used to include a file of binary data. +Note that L is easier to use and more +flexible. + +=item Create a small disk filled with a test pattern + + nbdkit ddrescue data="0x55*4096" + +This creates a disk containing 4096 0x55 bytes. See also +L. + +=item Create a 7 EB (exabyte) disk with one empty GPT-formatted partition: + +The ddrescue plugin supports huge sparse virtual disks, up to the maximum +size supported by nbdkit itself (S<2⁶³-1 bytes> = +S<8 exabytes - 1 byte>). These cause all sorts of problems for +software and so make excellent test cases. To partition such a huge +disk requires the GPT (GUID Partition Table) format since MBR is +limited to just 2 TB. + + nbdkit ddrescue data=" + @0x1c0 2 0 0xee 0xfe 0xff 0xff 0x01 0 0 0 0xff 0xff 0xff 0xff + @0x1fe 0x55 0xaa + @0x200 0x45 0x46 0x49 0x20 0x50 0x41 0x52 0x54 + 0 0 1 0 0x5c 0 0 0 + 0x9b 0xe5 0x6a 0xc5 0 0 0 0 1 0 0 0 0 0 0 0 + 0xff 0xff 0xff 0xff 0xff 0xff 0x37 0 0x22 0 0 0 0 0 0 0 + 0xde 0xff 0xff 0xff 0xff 0xff 0x37 0 + 0x72 0xb6 0x9e 0x0c 0x6b 0x76 0xb0 0x4f + 0xb3 0x94 0xb2 0xf1 0x61 0xec 0xdd 0x3c 2 0 0 0 0 0 0 0 + 0x80 0 0 0 0x80 0 0 0 0x79 0x8a 0xd0 0x7e 0 0 0 0 + @0x400 0xaf 0x3d 0xc6 0x0f 0x83 0x84 0x72 0x47 + 0x8e 0x79 0x3d 0x69 0xd8 0x47 0x7d 0xe4 + 0xd5 0x19 0x46 0x95 0xe3 0x82 0xa8 0x4c + 0x95 0x82 0x7a 0xbe 0x1c 0xfc 0x62 0x90 + 0x80 0 0 0 0 0 0 0 0x80 0xff 0xff 0xff 0xff 0xff 0x37 0 + 0 0 0 0 0 0 0 0 0x70 0 0x31 0 0 0 0 0 + @0x6fffffffffffbe00 + 0xaf 0x3d 0xc6 0x0f 0x83 0x84 0x72 0x47 + 0x8e 0x79 0x3d 0x69 0xd8 0x47 0x7d 0xe4 + 0xd5 0x19 0x46 0x95 0xe3 0x82 0xa8 0x4c + 0x95 0x82 0x7a 0xbe 0x1c 0xfc 0x62 0x90 + 0x80 0 0 0 0 0 0 0 0x80 0xff 0xff 0xff 0xff 0xff 0x37 0 + 0 0 0 0 0 0 0 0 0x70 0 0x31 0 0 0 0 0 + @0x6ffffffffffffe00 + 0x45 0x46 0x49 0x20 0x50 0x41 0x52 0x54 + 0 0 1 0 0x5c 0 0 0 + 0x6c 0x76 0xa1 0xa0 0 0 0 0 + 0xff 0xff 0xff 0xff 0xff 0xff 0x37 0 + 1 0 0 0 0 0 0 0 0x22 0 0 0 0 0 0 0 + 0xde 0xff 0xff 0xff 0xff 0xff 0x37 0 + 0x72 0xb6 0x9e 0x0c 0x6b 0x76 0xb0 0x4f + 0xb3 0x94 0xb2 0xf1 0x61 0xec 0xdd 0x3c + 0xdf 0xff 0xff 0xff 0xff 0xff 0x37 0 + 0x80 0 0 0 0x80 0 0 0 0x79 0x8a 0xd0 0x7e 0 0 0 0 + " size=7E + +=back + +=head1 PARAMETERS + +Exactly one of the C, C or C parameters must be +supplied. + +=over 4 + +=item BDATA + +Specify the disk data using a simple compact format. See +L below. + +=item BBASE64 + +The C parameter can be used to supply binary data encoded in +base64 on the command line. + +This is only supported if nbdkit was compiled with GnuTLS E 3.6.0. +You can find out by checking if: + + $ nbdkit ddrescue --dump-plugin + +contains: + + data_base64=yes + +=item BBINARY + +The C parameter can be used to supply raw binary data directly on +the command line. + +It is usually quite difficult to do this unless you are running nbdkit +from another program (see L). One particular +problem is that the data must not contain zero bytes (ie. C<\0>) since +those will be processed in C to mean the end of the string. In almost +all cases it is better to use base64 encoding or the custom C +format. + +=item BSIZE + +The data is truncated or extended to the size specified. + +This parameter is optional: If omitted the size is defined by the size +of the C, C or C parameter. + +=back + +=head1 DATA FORMAT + +The C parameter lets you specify small disk images in a simple, +compact format. It is a string containing a list of bytes which are +written into the disk image sequentially. You can move the virtual +offset where bytes are written using C<@offset>. + +For example: + + nbdkit ddrescue data="0 1 2 3 @0x1fe 0x55 0xaa" + +creates a 0x200 = 512 byte (1 sector) image containing the four bytes +C<0 1 2 3> at the start, and the two bytes C<0x55 0xaa> at the end of +the sector, with the remaining 506 bytes in the middle being all +zeroes. + +Fields in the string can be: + +=over 4 + +=item B<@>OFFSET + +Moves the current offset to C. The offset may be specified as +either decimal, octal (prefixed by C<0>) or hexadecimal (prefixed by +C<0x>). + +=item B>FILE + +Read the contents of F into the disk image at the current +offset. The offset is incremented by the size of the file. The +filename can be a relative or absolute path, but cannot contain +whitespace in the name. + +=item BYTE + +Write C at the current offset and advance the offset by 1 byte. +The byte may be specified as either decimal, octal (prefixed by C<0>) +or hexadecimal (prefixed by C<0x>). + +=item BYTE*N + +Repeat C C times (where C is a number). Note there must +not be any whitespace around the C<*> character. + +=back + +Whitespace between fields in the string is ignored. + +In the example above the size (512 bytes) is implied by the data. But +you could additionally use the C parameter to either truncate or +extend (with zeroes) the disk image. + +=head2 disk2data.pl script + +This script can convert from small disk images into the data format +described above. + +It is provided in the nbdkit sources. See +L + +=head1 FILES + +=over 4 + +=item F<$plugindir/nbdkit-data-plugin.so> + +The plugin. + +Use C to find the location of C<$plugindir>. + +=back + +=head1 VERSION + +C first appeared in nbdkit 1.6. + +=head1 SEE ALSO + +L, +L, +L, +L, +L, +L, +L, +L, +L, +L, +L, +L. + +=head1 AUTHORS + +Richard W.M. Jones + +=head1 COPYRIGHT + +Copyright (C) 2018 Red Hat Inc.