diff --git a/.gitignore b/.gitignore index eab3d979a506..6cabe62f30f8 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ *.swp .deps .libs +.dirstamp .DS_Store modules.order Makefile diff --git a/cmd/Makefile.am b/cmd/Makefile.am index 05f42a556bb5..04aa7c6333da 100644 --- a/cmd/Makefile.am +++ b/cmd/Makefile.am @@ -1,3 +1,3 @@ SUBDIRS = zfs zpool zdb zhack zinject zstreamdump ztest zpios SUBDIRS += mount_zfs fsck_zfs zvol_id vdev_id arcstat dbufstat zed -SUBDIRS += arc_summary +SUBDIRS += arc_summary raidz_test diff --git a/cmd/arc_summary/arc_summary.py b/cmd/arc_summary/arc_summary.py index 65b5c4dbd02f..9f6d8c1199f7 100755 --- a/cmd/arc_summary/arc_summary.py +++ b/cmd/arc_summary/arc_summary.py @@ -775,39 +775,13 @@ def _l2arc_summary(Kstat): def get_dmu_summary(Kstat): output = {} - zfetch_bogus_streams = Kstat["kstat.zfs.misc.zfetchstats.bogus_streams"] - zfetch_colinear_hits = Kstat["kstat.zfs.misc.zfetchstats.colinear_hits"] - zfetch_colinear_misses = \ - Kstat["kstat.zfs.misc.zfetchstats.colinear_misses"] zfetch_hits = Kstat["kstat.zfs.misc.zfetchstats.hits"] zfetch_misses = Kstat["kstat.zfs.misc.zfetchstats.misses"] - zfetch_reclaim_failures = \ - Kstat["kstat.zfs.misc.zfetchstats.reclaim_failures"] - zfetch_reclaim_successes = \ - Kstat["kstat.zfs.misc.zfetchstats.reclaim_successes"] - zfetch_streams_noresets = \ - Kstat["kstat.zfs.misc.zfetchstats.streams_noresets"] - zfetch_streams_resets = Kstat["kstat.zfs.misc.zfetchstats.streams_resets"] - zfetch_stride_hits = Kstat["kstat.zfs.misc.zfetchstats.stride_hits"] - zfetch_stride_misses = Kstat["kstat.zfs.misc.zfetchstats.stride_misses"] zfetch_access_total = (zfetch_hits + zfetch_misses) - zfetch_colinear_total = (zfetch_colinear_hits + zfetch_colinear_misses) - zfetch_health_count = (zfetch_bogus_streams) - zfetch_reclaim_total = (zfetch_reclaim_successes + zfetch_reclaim_failures) - zfetch_streams_total = (zfetch_streams_resets + zfetch_streams_noresets + - zfetch_bogus_streams) - zfetch_stride_total = (zfetch_stride_hits + zfetch_stride_misses) output['zfetch_access_total'] = zfetch_access_total if zfetch_access_total > 0: - - output['file_level_prefetch'] = {} - if zfetch_health_count > 0: - output['file_level_prefetch']['health'] = 'DEGRADED' - else: - output['file_level_prefetch']['health'] = 'HEALTHY' - output['dmu'] = {} output['dmu']['efficiency'] = {} output['dmu']['efficiency']['value'] = fHits(zfetch_access_total) @@ -820,57 +794,6 @@ def get_dmu_summary(Kstat): 'num': fHits(zfetch_misses), } - output['dmu']['colinear'] = {} - output['dmu']['colinear']['value'] = fHits(zfetch_colinear_total) - output['dmu']['colinear']['hit_ratio'] = { - 'per': fPerc(zfetch_colinear_hits, zfetch_colinear_total), - 'num': fHits(zfetch_colinear_hits), - } - output['dmu']['colinear']['miss_ratio'] = { - 'per': fPerc(zfetch_colinear_misses, zfetch_colinear_total), - 'num': fHits(zfetch_colinear_misses), - } - - output['dmu']['stride'] = {} - output['dmu']['stride']['value'] = fHits(zfetch_stride_total) - output['dmu']['stride']['hit_ratio'] = { - 'per': fPerc(zfetch_stride_hits, zfetch_stride_total), - 'num': fHits(zfetch_stride_hits), - } - output['dmu']['stride']['miss_ratio'] = { - 'per': fPerc(zfetch_stride_misses, zfetch_stride_total), - 'num': fHits(zfetch_stride_misses), - } - - output['dmu_misc'] = {} - if zfetch_health_count > 0: - output['dmu_misc']['status'] = "FAULTED" - else: - output['dmu_misc']['status'] = "" - - output['dmu_misc']['reclaim'] = {} - output['dmu_misc']['reclaim']['value'] = fHits(zfetch_reclaim_total) - output['dmu_misc']['reclaim']['successes'] = { - 'per': fPerc(zfetch_reclaim_successes, zfetch_reclaim_total), - 'num': fHits(zfetch_reclaim_successes), - } - output['dmu_misc']['reclaim']['failure'] = { - 'per': fPerc(zfetch_reclaim_failures, zfetch_reclaim_total), - 'num': fHits(zfetch_reclaim_failures), - } - - output['dmu_misc']['streams'] = {} - output['dmu_misc']['streams']['value'] = fHits(zfetch_streams_total) - output['dmu_misc']['streams']['plus_resets'] = { - 'per': fPerc(zfetch_streams_resets, zfetch_streams_total), - 'num': fHits(zfetch_streams_resets), - } - output['dmu_misc']['streams']['neg_resets'] = { - 'per': fPerc(zfetch_streams_noresets, zfetch_streams_total), - 'num': fHits(zfetch_streams_noresets), - } - output['dmu_misc']['streams']['bogus'] = fHits(zfetch_bogus_streams) - return output @@ -879,11 +802,7 @@ def _dmu_summary(Kstat): arc = get_dmu_summary(Kstat) if arc['zfetch_access_total'] > 0: - sys.stdout.write("File-Level Prefetch: (%s)" % - arc['file_level_prefetch']['health']) - sys.stdout.write("\n") - - sys.stdout.write("DMU Efficiency:\t\t\t\t\t%s\n" % + sys.stdout.write("DMU Prefetch Efficiency:\t\t\t\t\t%s\n" % arc['dmu']['efficiency']['value']) sys.stdout.write("\tHit Ratio:\t\t\t%s\t%s\n" % ( arc['dmu']['efficiency']['hit_ratio']['per'], @@ -898,70 +817,6 @@ def _dmu_summary(Kstat): sys.stdout.write("\n") - sys.stdout.write("\tColinear:\t\t\t\t%s\n" % - arc['dmu']['colinear']['value']) - sys.stdout.write("\t Hit Ratio:\t\t\t%s\t%s\n" % ( - arc['dmu']['colinear']['hit_ratio']['per'], - arc['dmu']['colinear']['hit_ratio']['num'], - ) - ) - - sys.stdout.write("\t Miss Ratio:\t\t\t%s\t%s\n" % ( - arc['dmu']['colinear']['miss_ratio']['per'], - arc['dmu']['colinear']['miss_ratio']['num'], - ) - ) - - sys.stdout.write("\n") - - sys.stdout.write("\tStride:\t\t\t\t\t%s\n" % - arc['dmu']['stride']['value']) - sys.stdout.write("\t Hit Ratio:\t\t\t%s\t%s\n" % ( - arc['dmu']['stride']['hit_ratio']['per'], - arc['dmu']['stride']['hit_ratio']['num'], - ) - ) - - sys.stdout.write("\t Miss Ratio:\t\t\t%s\t%s\n" % ( - arc['dmu']['stride']['miss_ratio']['per'], - arc['dmu']['stride']['miss_ratio']['num'], - ) - ) - - sys.stdout.write("\n") - sys.stdout.write("DMU Misc: %s\n" % arc['dmu_misc']['status']) - - sys.stdout.write("\tReclaim:\t\t\t\t%s\n" % - arc['dmu_misc']['reclaim']['value']) - sys.stdout.write("\t Successes:\t\t\t%s\t%s\n" % ( - arc['dmu_misc']['reclaim']['successes']['per'], - arc['dmu_misc']['reclaim']['successes']['num'], - ) - ) - - sys.stdout.write("\t Failures:\t\t\t%s\t%s\n" % ( - arc['dmu_misc']['reclaim']['failure']['per'], - arc['dmu_misc']['reclaim']['failure']['num'], - ) - ) - - sys.stdout.write("\n\tStreams:\t\t\t\t%s\n" % - arc['dmu_misc']['streams']['value']) - sys.stdout.write("\t +Resets:\t\t\t%s\t%s\n" % ( - arc['dmu_misc']['streams']['plus_resets']['per'], - arc['dmu_misc']['streams']['plus_resets']['num'], - ) - ) - - sys.stdout.write("\t -Resets:\t\t\t%s\t%s\n" % ( - arc['dmu_misc']['streams']['neg_resets']['per'], - arc['dmu_misc']['streams']['neg_resets']['num'], - ) - ) - - sys.stdout.write("\t Bogus:\t\t\t\t%s\n" % - arc['dmu_misc']['streams']['bogus']) - def get_vdev_summary(Kstat): output = {} diff --git a/cmd/raidz_test/.gitignore b/cmd/raidz_test/.gitignore new file mode 100644 index 000000000000..f8b83d9cce03 --- /dev/null +++ b/cmd/raidz_test/.gitignore @@ -0,0 +1 @@ +/raidz_test diff --git a/cmd/raidz_test/Makefile.am b/cmd/raidz_test/Makefile.am new file mode 100644 index 000000000000..23ad08ad9925 --- /dev/null +++ b/cmd/raidz_test/Makefile.am @@ -0,0 +1,21 @@ +include $(top_srcdir)/config/Rules.am + +AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN) +AM_CPPFLAGS += -DDEBUG + +DEFAULT_INCLUDES += \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/lib/libspl/include + +bin_PROGRAMS = raidz_test + +raidz_test_SOURCES = \ + raidz_test.h \ + raidz_test.c \ + raidz_bench.c + +raidz_test_LDADD = \ + $(top_builddir)/lib/libuutil/libuutil.la \ + $(top_builddir)/lib/libzpool/libzpool.la + +raidz_test_LDADD += -lm -ldl diff --git a/cmd/raidz_test/raidz_bench.c b/cmd/raidz_test/raidz_bench.c new file mode 100644 index 000000000000..f1710ccc7c4b --- /dev/null +++ b/cmd/raidz_test/raidz_bench.c @@ -0,0 +1,227 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "raidz_test.h" + +#define GEN_BENCH_MEMORY (((uint64_t)1ULL)<<32) +#define REC_BENCH_MEMORY (((uint64_t)1ULL)<<29) +#define BENCH_ASHIFT 12 +#define MIN_CS_SHIFT BENCH_ASHIFT +#define MAX_CS_SHIFT SPA_MAXBLOCKSHIFT + +static zio_t zio_bench; +static raidz_map_t *rm_bench; +static size_t max_data_size = SPA_MAXBLOCKSIZE; + +static void +bench_init_raidz_map(void) +{ + zio_bench.io_offset = 0; + zio_bench.io_size = max_data_size; + + /* + * To permit larger column sizes these have to be done + * allocated using aligned alloc instead of zio_data_buf_alloc + */ + zio_bench.io_data = raidz_alloc(max_data_size); + + init_zio_data(&zio_bench); +} + +static void +bench_fini_raidz_maps(void) +{ + /* tear down golden zio */ + raidz_free(zio_bench.io_data, max_data_size); + bzero(&zio_bench, sizeof (zio_t)); +} + +static inline void +run_gen_bench_impl(const char *impl) +{ + int fn, ncols; + uint64_t ds, iter_cnt, iter, disksize; + hrtime_t start; + double elapsed, d_bw; + + /* Benchmark generate functions */ + for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) { + + for (ds = MIN_CS_SHIFT; ds <= MAX_CS_SHIFT; ds++) { + /* create suitable raidz_map */ + ncols = rto_opts.rto_dcols + fn + 1; + zio_bench.io_size = 1ULL << ds; + rm_bench = vdev_raidz_map_alloc(&zio_bench, + BENCH_ASHIFT, ncols, fn+1); + + /* estimate iteration count */ + iter_cnt = GEN_BENCH_MEMORY; + iter_cnt /= zio_bench.io_size; + + start = gethrtime(); + for (iter = 0; iter < iter_cnt; iter++) + vdev_raidz_generate_parity(rm_bench); + elapsed = NSEC2SEC((double) (gethrtime() - start)); + + disksize = (1ULL << ds) / rto_opts.rto_dcols; + d_bw = (double)iter_cnt * (double)disksize; + d_bw /= (1024.0 * 1024.0 * elapsed); + + LOG(D_ALL, "%10s, %8s, %zu, %10llu, %lf, %lf, %u\n", + impl, + raidz_gen_name[fn], + rto_opts.rto_dcols, + (1ULL< +#include +#include +#include +#include +#include +#include +#include +#include +#include "raidz_test.h" + +static int *rand_data; +raidz_test_opts_t rto_opts; + +static char gdb[256]; +static const char gdb_tmpl[] = "gdb -ex \"set pagination 0\" -p %d"; + +static void sig_handler(int signo) +{ + struct sigaction action; + /* + * Restore default action and re-raise signal so SIGSEGV and + * SIGABRT can trigger a core dump. + */ + action.sa_handler = SIG_DFL; + sigemptyset(&action.sa_mask); + action.sa_flags = 0; + (void) sigaction(signo, &action, NULL); + + if (rto_opts.rto_gdb) + if (system(gdb)); + + raise(signo); +} + +static void print_opts(raidz_test_opts_t *opts, boolean_t force) +{ + char *verbose; + switch (opts->rto_v) { + case 0: + verbose = "no"; + break; + case 1: + verbose = "info"; + break; + default: + verbose = "debug"; + break; + } + + if (force || opts->rto_v >= D_INFO) { + (void) fprintf(stdout, DBLSEP "Running with options:\n" + " (-a) zio ashift : %zu\n" + " (-o) zio offset : 1 << %zu\n" + " (-d) number of raidz data columns : %zu\n" + " (-s) size of DATA : 1 << %zu\n" + " (-S) sweep parameters : %s \n" + " (-v) verbose : %s \n\n", + opts->rto_ashift, /* -a */ + ilog2(opts->rto_offset), /* -o */ + opts->rto_dcols, /* -d */ + ilog2(opts->rto_dsize), /* -s */ + opts->rto_sweep ? "yes" : "no", /* -S */ + verbose /* -v */ + ); + } +} + +static void usage(boolean_t requested) +{ + const raidz_test_opts_t *o = &rto_opts_defaults; + + FILE *fp = requested ? stdout : stderr; + + (void) fprintf(fp, "Usage:\n" + "\t[-a zio ashift (default: %zu)]\n" + "\t[-o zio offset, exponent radix 2 (default: %zu)]\n" + "\t[-d number of raidz data columns (default: %zu)]\n" + "\t[-s zio size, exponent radix 2 (default: %zu)]\n" + "\t[-S parameter sweep (default: %s)]\n" + "\t[-t timeout for parameter sweep test]\n" + "\t[-B benchmark all raidz implementations]\n" + "\t[-v increase verbosity (default: %zu)]\n" + "\t[-h (print help)]\n" + "\t[-T test the test, see if failure would be detected]\n" + "\t[-D debug (attach gdb on SIGSEGV)]\n" + "", + o->rto_ashift, /* -a */ + ilog2(o->rto_offset), /* -o */ + o->rto_dcols, /* -d */ + ilog2(o->rto_dsize), /* -s */ + rto_opts.rto_sweep ? "yes" : "no", /* -S */ + o->rto_v /* -d */ + ); + + exit(requested ? 0 : 1); +} + +static void process_options(int argc, char **argv) +{ + size_t value; + int opt; + + raidz_test_opts_t *o = &rto_opts; + + bcopy(&rto_opts_defaults, o, sizeof (*o)); + + while ((opt = getopt(argc, argv, "TDBSvha:o:d:s:t:")) != -1) { + value = 0; + + switch (opt) { + case 'a': + value = strtoull(optarg, NULL, 0); + o->rto_ashift = MIN(13, MAX(9, value)); + break; + case 'o': + value = strtoull(optarg, NULL, 0); + o->rto_offset = ((1ULL << MIN(12, value)) >> 9) << 9; + break; + case 'd': + value = strtoull(optarg, NULL, 0); + o->rto_dcols = MIN(255, MAX(1, value)); + break; + case 's': + value = strtoull(optarg, NULL, 0); + o->rto_dsize = 1ULL << MIN(SPA_MAXBLOCKSHIFT, + MAX(SPA_MINBLOCKSHIFT, value)); + break; + case 't': + value = strtoull(optarg, NULL, 0); + o->rto_sweep_timeout = value; + break; + case 'v': + o->rto_v++; + break; + case 'S': + o->rto_sweep = 1; + break; + case 'B': + o->rto_benchmark = 1; + break; + case 'D': + o->rto_gdb = 1; + break; + case 'T': + o->rto_sanity = 1; + break; + case 'h': + usage(B_TRUE); + break; + case '?': + default: + usage(B_FALSE); + break; + } + } +} + +#define DATA_COL(rm, i) ((rm)->rm_col[raidz_parity(rm) + (i)].rc_data) +#define DATA_COL_SIZE(rm, i) ((rm)->rm_col[raidz_parity(rm) + (i)].rc_size) + +#define CODE_COL(rm, i) ((rm)->rm_col[(i)].rc_data) +#define CODE_COL_SIZE(rm, i) ((rm)->rm_col[(i)].rc_size) + +static int +cmp_code(raidz_test_opts_t *opts, const raidz_map_t *rm, const int parity) +{ + int i, ret = 0; + + VERIFY(parity >= 1 && parity <= 3); + + for (i = 0; i < parity; i++) { + if (0 != memcmp(CODE_COL(rm, i), CODE_COL(opts->rm_golden, i), + CODE_COL_SIZE(rm, i))) { + ret++; + + LOG_OPT(D_DEBUG, opts, + "\nParity block [%d] different!\n", i); + } + } + return (ret); +} + +static int +cmp_data(raidz_test_opts_t *opts, raidz_map_t *rm) +{ + int i, ret = 0; + int dcols = opts->rm_golden->rm_cols - raidz_parity(opts->rm_golden); + + for (i = 0; i < dcols; i++) { + if (0 != memcmp(DATA_COL(opts->rm_golden, i), DATA_COL(rm, i), + DATA_COL_SIZE(opts->rm_golden, i))) { + ret++; + + LOG_OPT(D_DEBUG, opts, + "\nData block [%d] different!\n", i); + } + } + return (ret); +} + +static void +corrupt_colums(raidz_map_t *rm, const int *tgts, const int cnt) +{ + int i; + int *dst; + raidz_col_t *col; + + for (i = 0; i < cnt; i++) { + col = &rm->rm_col[tgts[i]]; + dst = col->rc_data; + for (i = 0; i < col->rc_size / sizeof (int); i++) + dst[i] = rand(); + } +} + +void +init_zio_data(zio_t *zio) +{ + int i; + int *dst = (int *) zio->io_data; + + for (i = 0; i < zio->io_size / sizeof (int); i++) { + dst[i] = rand_data[i]; + } +} + +static void +fini_raidz_map(zio_t **zio, raidz_map_t **rm) +{ + vdev_raidz_map_free(*rm); + raidz_free((*zio)->io_data, (*zio)->io_size); + umem_free(*zio, sizeof (zio_t)); + + *zio = NULL; + *rm = NULL; +} + +static int +init_raidz_golden_map(raidz_test_opts_t *opts, const int parity) +{ + int err = 0; + zio_t *zio_test; + raidz_map_t *rm_test; + const size_t total_ncols = opts->rto_dcols + parity; + + if (opts->rm_golden) { + fini_raidz_map(&opts->zio_golden, &opts->rm_golden); + } + + opts->zio_golden = umem_zalloc(sizeof (zio_t), UMEM_NOFAIL); + zio_test = umem_zalloc(sizeof (zio_t), UMEM_NOFAIL); + + opts->zio_golden->io_offset = zio_test->io_offset = opts->rto_offset; + opts->zio_golden->io_size = zio_test->io_size = opts->rto_dsize; + + opts->zio_golden->io_data = raidz_alloc(opts->rto_dsize); + zio_test->io_data = raidz_alloc(opts->rto_dsize); + + init_zio_data(opts->zio_golden); + init_zio_data(zio_test); + + VERIFY0(vdev_raidz_impl_set("original")); + + opts->rm_golden = vdev_raidz_map_alloc(opts->zio_golden, + opts->rto_ashift, total_ncols, parity); + rm_test = vdev_raidz_map_alloc(zio_test, + opts->rto_ashift, total_ncols, parity); + + VERIFY(opts->zio_golden); + VERIFY(opts->rm_golden); + + vdev_raidz_generate_parity(opts->rm_golden); + vdev_raidz_generate_parity(rm_test); + + /* sanity check */ + err |= cmp_data(opts, rm_test); + err |= cmp_code(opts, rm_test, parity); + + if (err) + ERR("initializing the golden copy ... [FAIL]!\n"); + + /* tear down raidz_map of test zio */ + fini_raidz_map(&zio_test, &rm_test); + + return (err); +} + +static raidz_map_t * +init_raidz_map(raidz_test_opts_t *opts, zio_t **zio, const int parity) +{ + raidz_map_t *rm = NULL; + const size_t alloc_dsize = opts->rto_dsize; + const size_t total_ncols = opts->rto_dcols + parity; + const int ccols[] = { 0, 1, 2 }; + + VERIFY(zio); + VERIFY(parity <= 3 && parity >= 1); + + *zio = umem_zalloc(sizeof (zio_t), UMEM_NOFAIL); + + (*zio)->io_offset = 0; + (*zio)->io_size = alloc_dsize; + (*zio)->io_data = raidz_alloc(alloc_dsize); + init_zio_data(*zio); + + rm = vdev_raidz_map_alloc(*zio, opts->rto_ashift, + total_ncols, parity); + VERIFY(rm); + + /* Make sure code columns are destroyed */ + corrupt_colums(rm, ccols, parity); + + return (rm); +} + +static int +run_gen_check(raidz_test_opts_t *opts) +{ + char **impl_name; + int fn, err = 0; + zio_t *zio_test; + raidz_map_t *rm_test; + + err = init_raidz_golden_map(opts, PARITY_PQR); + if (0 != err) + return (err); + + LOG(D_INFO, DBLSEP); + LOG(D_INFO, "Testing parity generation...\n"); + + for (impl_name = (char **)raidz_impl_names+1; *impl_name != NULL; + impl_name++) { + + LOG(D_INFO, SEP); + LOG(D_INFO, "\tTesting [%s] implementation...", *impl_name); + + if (0 != vdev_raidz_impl_set(*impl_name)) { + LOG(D_INFO, "[SKIP]\n"); + continue; + } else { + LOG(D_INFO, "[SUPPORTED]\n"); + } + + for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) { + + /* create suitable raidz_map */ + rm_test = init_raidz_map(opts, &zio_test, fn+1); + VERIFY(rm_test); + + LOG(D_INFO, "\t\tTesting method [%s] ...", + raidz_gen_name[fn]); + + if (!opts->rto_sanity) + vdev_raidz_generate_parity(rm_test); + + if (cmp_code(opts, rm_test, fn+1) != 0) { + LOG(D_INFO, "[FAIL]\n"); + err++; + } else + LOG(D_INFO, "[PASS]\n"); + + fini_raidz_map(&zio_test, &rm_test); + } + } + + fini_raidz_map(&opts->zio_golden, &opts->rm_golden); + + return (err); +} + +static int +run_rec_check_impl(raidz_test_opts_t *opts, raidz_map_t *rm, const int fn) +{ + int x0, x1, x2; + int tgtidx[3]; + int err = 0; + static const int rec_tgts[7][3] = { + {1, 2, 3}, /* rec_p: bad QR & D[0] */ + {0, 2, 3}, /* rec_q: bad PR & D[0] */ + {0, 1, 3}, /* rec_r: bad PQ & D[0] */ + {2, 3, 4}, /* rec_pq: bad R & D[0][1] */ + {1, 3, 4}, /* rec_pr: bad Q & D[0][1] */ + {0, 3, 4}, /* rec_qr: bad P & D[0][1] */ + {3, 4, 5} /* rec_pqr: bad & D[0][1][2] */ + }; + + memcpy(tgtidx, rec_tgts[fn], sizeof (tgtidx)); + + if (fn < RAIDZ_REC_PQ) { + /* can reconstruct 1 failed data disk */ + for (x0 = 0; x0 < opts->rto_dcols; x0++) { + if (x0 >= rm->rm_cols - raidz_parity(rm)) + continue; + + LOG(D_DEBUG, "[%d] ", x0); + + tgtidx[2] = x0 + raidz_parity(rm); + + corrupt_colums(rm, tgtidx+2, 1); + + if (!opts->rto_sanity) + vdev_raidz_reconstruct(rm, tgtidx, 3); + + if (cmp_data(opts, rm) != 0) { + err++; + LOG(D_DEBUG, "\nREC D[%d]... [FAIL]\n", x0); + } + } + + } else if (fn < RAIDZ_REC_PQR) { + /* can reconstruct 2 failed data disk */ + for (x0 = 0; x0 < opts->rto_dcols; x0++) { + if (x0 >= rm->rm_cols - raidz_parity(rm)) + continue; + for (x1 = x0 + 1; x1 < opts->rto_dcols; x1++) { + if (x1 >= rm->rm_cols - raidz_parity(rm)) + continue; + + LOG(D_DEBUG, "[%d %d] ", x0, x1); + + tgtidx[1] = x0 + raidz_parity(rm); + tgtidx[2] = x1 + raidz_parity(rm); + + corrupt_colums(rm, tgtidx+1, 2); + + if (!opts->rto_sanity) + vdev_raidz_reconstruct(rm, tgtidx, 3); + + if (cmp_data(opts, rm) != 0) { + err++; + LOG(D_DEBUG, "\nREC D[%d %d]... " + "[FAIL]\n", x0, x1); + } + } + } + } else { + /* can reconstruct 3 failed data disk */ + for (x0 = 0; + x0 < opts->rto_dcols; x0++) { + if (x0 >= rm->rm_cols - raidz_parity(rm)) + continue; + for (x1 = x0 + 1; + x1 < opts->rto_dcols; x1++) { + if (x1 >= rm->rm_cols - raidz_parity(rm)) + continue; + for (x2 = x1 + 1; + x2 < opts->rto_dcols; x2++) { + if (x2 >= + rm->rm_cols - raidz_parity(rm)) + continue; + + LOG(D_DEBUG, "[%d %d %d]", x0, x1, x2); + + tgtidx[0] = x0 + raidz_parity(rm); + tgtidx[1] = x1 + raidz_parity(rm); + tgtidx[2] = x2 + raidz_parity(rm); + + corrupt_colums(rm, tgtidx, 3); + + if (!opts->rto_sanity) + vdev_raidz_reconstruct(rm, + tgtidx, 3); + + if (cmp_data(opts, rm) != 0) { + err++; + LOG(D_DEBUG, + "\nREC D[%d %d %d]... " + "[FAIL]\n", x0, x1, x2); + } + } + } + } + } + return (err); +} + +static int +run_rec_check(raidz_test_opts_t *opts) +{ + char **impl_name; + unsigned fn, err = 0; + zio_t *zio_test; + raidz_map_t *rm_test; + + err = init_raidz_golden_map(opts, PARITY_PQR); + if (0 != err) + return (err); + + LOG(D_INFO, DBLSEP); + LOG(D_INFO, "Testing data reconstruction...\n"); + + for (impl_name = (char **)raidz_impl_names+1; *impl_name != NULL; + impl_name++) { + + LOG(D_INFO, SEP); + LOG(D_INFO, "\tTesting [%s] implementation...", *impl_name); + + if (vdev_raidz_impl_set(*impl_name) != 0) { + LOG(D_INFO, "[SKIP]\n"); + continue; + } else + LOG(D_INFO, "[SUPPORTED]\n"); + + + /* create suitable raidz_map */ + rm_test = init_raidz_map(opts, &zio_test, PARITY_PQR); + /* generate parity */ + vdev_raidz_generate_parity(rm_test); + + for (fn = 0; fn < RAIDZ_REC_NUM; fn++) { + + LOG(D_INFO, "\t\tTesting method [%s] ...", + raidz_rec_name[fn]); + + if (run_rec_check_impl(opts, rm_test, fn) != 0) { + LOG(D_INFO, "[FAIL]\n"); + err++; + + } else + LOG(D_INFO, "[PASS]\n"); + + } + /* tear down test raidz_map */ + fini_raidz_map(&zio_test, &rm_test); + } + + fini_raidz_map(&opts->zio_golden, &opts->rm_golden); + + return (err); +} + +static int +run_test(raidz_test_opts_t *opts) +{ + int err = 0; + + if (opts == NULL) + opts = &rto_opts; + + print_opts(opts, B_FALSE); + + err |= run_gen_check(opts); + err |= run_rec_check(opts); + + return (err); +} + +#define SWEEP_RUNNING 0 +#define SWEEP_FINISHED 1 +#define SWEEP_ERROR 2 +#define SWEEP_TIMEOUT 3 + +static int sweep_state = 0; +static raidz_test_opts_t failed_opts; + +static kmutex_t sem_mtx; +static kcondvar_t sem_cv; +static int max_free_slots; +static int free_slots; + +static void +sweep_thread(void *arg) +{ + int err = 0; + raidz_test_opts_t *opts = (raidz_test_opts_t *) arg; + VERIFY(opts != NULL); + + err = run_test(opts); + + if (rto_opts.rto_sanity) { + /* 25% chance that a sweep test fails */ + if (rand() < (RAND_MAX/4)) + err = 1; + } + + if (0 != err) { + mutex_enter(&sem_mtx); + memcpy(&failed_opts, opts, sizeof (raidz_test_opts_t)); + sweep_state = SWEEP_ERROR; + mutex_exit(&sem_mtx); + } + + umem_free(opts, sizeof (raidz_test_opts_t)); + + /* signal the next thread */ + mutex_enter(&sem_mtx); + free_slots++; + cv_signal(&sem_cv); + mutex_exit(&sem_mtx); + + thread_exit(); +} + +static int +run_sweep(void) +{ + static const size_t dcols_v[] = { 1, 2, 3, 4, 5, 6, 7, 8 }; + static const size_t ashift_v[] = { 9, 12 }; + static const size_t offset_cnt = 4; + static const size_t size_v[] = { 1 << 9, 21 * (1 << 9), 13 * (1 << 12), + 1 << 17, (1 << 20) - (1 << 12), SPA_MAXBLOCKSIZE }; + + (void) setvbuf(stdout, NULL, _IONBF, 0); + + ulong_t total_comb = ARRAY_SIZE(size_v) * ARRAY_SIZE(ashift_v) * + ARRAY_SIZE(dcols_v) * offset_cnt; + ulong_t tried_comb = 0; + hrtime_t time_diff, start_time = gethrtime(); + raidz_test_opts_t *opts; + int a, d, o, s; + + max_free_slots = free_slots = MAX(2, boot_ncpus); + + mutex_init(&sem_mtx, NULL, MUTEX_DEFAULT, NULL); + cv_init(&sem_cv, NULL, CV_DEFAULT, NULL); + + for (s = 0; s < ARRAY_SIZE(size_v); s++) + for (a = 0; a < ARRAY_SIZE(ashift_v); a++) + for (o = 0; o < offset_cnt; o++) + for (d = 0; d < ARRAY_SIZE(dcols_v); d++) { + + if ((size_v[s] < (1 << ashift_v[a]) * o) || + (size_v[s] < (1 << ashift_v[a]) * dcols_v[d])) { + total_comb--; + continue; + } + + if (++tried_comb % 20 == 0) + LOG(D_ALL, "%lu/%lu... ", tried_comb, total_comb); + + /* wait for signal to start new thread */ + mutex_enter(&sem_mtx); + while (cv_timedwait_sig(&sem_cv, &sem_mtx, + ddi_get_lbolt() + hz)) { + + /* check if should stop the test (timeout) */ + time_diff = (gethrtime() - start_time) / NANOSEC; + if (rto_opts.rto_sweep_timeout > 0 && + time_diff >= rto_opts.rto_sweep_timeout) { + sweep_state = SWEEP_TIMEOUT; + mutex_exit(&sem_mtx); + goto exit; + } + + /* check if should stop the test (error) */ + if (sweep_state != SWEEP_RUNNING) { + mutex_exit(&sem_mtx); + goto exit; + } + + /* exit loop if a slot is available */ + if (free_slots > 0) { + break; + } + } + + free_slots--; + mutex_exit(&sem_mtx); + + opts = umem_zalloc(sizeof (raidz_test_opts_t), UMEM_NOFAIL); + opts->rto_ashift = ashift_v[a]; + opts->rto_dcols = dcols_v[d]; + opts->rto_offset = (1 << ashift_v[a]) * o; + opts->rto_dsize = size_v[s]; + opts->rto_v = 0; /* be quiet */ + + VERIFY3P(zk_thread_create(NULL, 0, + (thread_func_t) sweep_thread, + (void *) opts, TS_RUN, NULL, 0, 0, + PTHREAD_CREATE_JOINABLE), !=, NULL); + } + +exit: + LOG(D_ALL, "\nWaiting for test threads to finish...\n"); + mutex_enter(&sem_mtx); + VERIFY(free_slots <= max_free_slots); + while (free_slots < max_free_slots) { + (void) cv_wait(&sem_cv, &sem_mtx); + } + mutex_exit(&sem_mtx); + + if (sweep_state == SWEEP_ERROR) { + ERR("Sweep test failed! Failed option: \n"); + print_opts(&failed_opts, B_TRUE); + } else { + if (sweep_state == SWEEP_TIMEOUT) + LOG(D_ALL, "Test timeout (%lus). Stopping...\n", + (ulong_t)rto_opts.rto_sweep_timeout); + + LOG(D_ALL, "Sweep test succeeded on %lu raidz maps!\n", + (ulong_t)tried_comb); + } + + return (sweep_state == SWEEP_ERROR ? SWEEP_ERROR : 0); +} + +int +main(int argc, char **argv) +{ + size_t i; + struct sigaction action; + int err = 0; + + /* init gdb string early */ + (void) sprintf(gdb, gdb_tmpl, getpid()); + + action.sa_handler = sig_handler; + sigemptyset(&action.sa_mask); + action.sa_flags = 0; + + if (sigaction(SIGSEGV, &action, NULL) < 0) { + ERR("raidz_test: cannot catch SIGSEGV: %s.\n", strerror(errno)); + exit(EXIT_FAILURE); + } + + (void) setvbuf(stdout, NULL, _IOLBF, 0); + + dprintf_setup(&argc, argv); + + process_options(argc, argv); + + kernel_init(FREAD); + + /* setup random data because rand() is not reentrant */ + rand_data = (int *) umem_alloc(SPA_MAXBLOCKSIZE, UMEM_NOFAIL); + srand((unsigned)time(NULL) * getpid()); + for (i = 0; i < SPA_MAXBLOCKSIZE / sizeof (int); i++) + rand_data[i] = rand(); + + mprotect(rand_data, SPA_MAXBLOCKSIZE, PROT_READ); + + if (rto_opts.rto_benchmark) { + run_raidz_benchmark(); + } else if (rto_opts.rto_sweep) { + err = run_sweep(); + } else { + err = run_test(NULL); + } + + umem_free(rand_data, SPA_MAXBLOCKSIZE); + kernel_fini(); + + return (err); +} diff --git a/cmd/raidz_test/raidz_test.h b/cmd/raidz_test/raidz_test.h new file mode 100644 index 000000000000..c3579dab0297 --- /dev/null +++ b/cmd/raidz_test/raidz_test.h @@ -0,0 +1,107 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. + */ + +#ifndef RAIDZ_TEST_H +#define RAIDZ_TEST_H + +#include + +static const char *raidz_impl_names[] = { + "original", + "scalar", + "sse2", + "ssse3", + "avx2", + NULL +}; + +typedef struct raidz_test_opts { + size_t rto_ashift; + size_t rto_offset; + size_t rto_dcols; + size_t rto_dsize; + size_t rto_v; + size_t rto_sweep; + size_t rto_sweep_timeout; + size_t rto_benchmark; + size_t rto_sanity; + size_t rto_gdb; + + zio_t *zio_golden; + raidz_map_t *rm_golden; +} raidz_test_opts_t; + +static const raidz_test_opts_t rto_opts_defaults = { + .rto_ashift = 9, + .rto_offset = 1ULL << 0, + .rto_dcols = 8, + .rto_dsize = 1<<19, + .rto_v = 0, + .rto_sweep = 0, + .rto_benchmark = 0, + .rto_sanity = 0, + .rto_gdb = 0 +}; + +extern raidz_test_opts_t rto_opts; + +static inline size_t ilog2(size_t a) +{ + return (a > 1 ? 1 + ilog2(a >> 1) : 0); +} + + +#define D_ALL 0 +#define D_INFO 1 +#define D_DEBUG 2 + +#define LOG(lvl, a...) \ +{ \ + if (rto_opts.rto_v >= lvl) \ + (void) fprintf(stdout, a); \ +} \ + +#define LOG_OPT(lvl, opt, a...) \ +{ \ + if (opt->rto_v >= lvl) \ + (void) fprintf(stdout, a); \ +} \ + +#define ERR(a...) (void) fprintf(stderr, a) + + +#define DBLSEP "================\n" +#define SEP "----------------\n" + + +#define raidz_alloc(size) zio_data_buf_alloc(size) +#define raidz_free(p, size) zio_data_buf_free(p, size) + + +void init_zio_data(zio_t *zio); + +void run_raidz_benchmark(void); + +#endif /* RAIDZ_TEST_H */ diff --git a/cmd/zdb/zdb.c b/cmd/zdb/zdb.c index cbc98d24c2f8..e35d62cab9d1 100644 --- a/cmd/zdb/zdb.c +++ b/cmd/zdb/zdb.c @@ -60,7 +60,6 @@ #include #include #include -#undef ZFS_MAXNAMELEN #include #define ZDB_COMPRESS_NAME(idx) ((idx) < ZIO_COMPRESS_FUNCTIONS ? \ @@ -172,7 +171,7 @@ usage(void) "-e to specify path to vdev dir\n"); (void) fprintf(stderr, " -x -- " "dump all read blocks into specified directory\n"); - (void) fprintf(stderr, " -P print numbers in parseable form\n"); + (void) fprintf(stderr, " -P print numbers in parsable form\n"); (void) fprintf(stderr, " -t -- highest txg to use when " "searching for uberblocks\n"); (void) fprintf(stderr, " -I -- " @@ -1880,15 +1879,15 @@ dump_object(objset_t *os, uint64_t object, int verbosity, int *print_header) dnode_t *dn; void *bonus = NULL; size_t bsize = 0; - char iblk[32], dblk[32], lsize[32], asize[32], fill[32]; + char iblk[32], dblk[32], lsize[32], asize[32], fill[32], dnsize[32]; char bonus_size[32]; char aux[50]; int error; if (*print_header) { - (void) printf("\n%10s %3s %5s %5s %5s %5s %6s %s\n", - "Object", "lvl", "iblk", "dblk", "dsize", "lsize", - "%full", "type"); + (void) printf("\n%10s %3s %5s %5s %5s %6s %5s %6s %s\n", + "Object", "lvl", "iblk", "dblk", "dsize", "dnsize", + "lsize", "%full", "type"); *print_header = 0; } @@ -1910,6 +1909,7 @@ dump_object(objset_t *os, uint64_t object, int verbosity, int *print_header) zdb_nicenum(doi.doi_max_offset, lsize); zdb_nicenum(doi.doi_physical_blocks_512 << 9, asize); zdb_nicenum(doi.doi_bonus_size, bonus_size); + zdb_nicenum(doi.doi_dnodesize, dnsize); (void) sprintf(fill, "%6.2f", 100.0 * doi.doi_fill_count * doi.doi_data_block_size / (object == 0 ? DNODES_PER_BLOCK : 1) / doi.doi_max_offset); @@ -1926,13 +1926,13 @@ dump_object(objset_t *os, uint64_t object, int verbosity, int *print_header) ZDB_COMPRESS_NAME(doi.doi_compress)); } - (void) printf("%10lld %3u %5s %5s %5s %5s %6s %s%s\n", + (void) printf("%10lld %3u %5s %5s %5s %6s %5s %6s %s%s\n", (u_longlong_t)object, doi.doi_indirection, iblk, dblk, - asize, lsize, fill, ZDB_OT_NAME(doi.doi_type), aux); + asize, dnsize, lsize, fill, ZDB_OT_NAME(doi.doi_type), aux); if (doi.doi_bonus_type != DMU_OT_NONE && verbosity > 3) { - (void) printf("%10s %3s %5s %5s %5s %5s %6s %s\n", - "", "", "", "", "", bonus_size, "bonus", + (void) printf("%10s %3s %5s %5s %5s %5s %5s %6s %s\n", + "", "", "", "", "", "", bonus_size, "bonus", ZDB_OT_NAME(doi.doi_bonus_type)); } @@ -2004,7 +2004,7 @@ dump_dir(objset_t *os) uint64_t refdbytes, usedobjs, scratch; char numbuf[32]; char blkbuf[BP_SPRINTF_LEN + 20]; - char osname[MAXNAMELEN]; + char osname[ZFS_MAX_DATASET_NAME_LEN]; char *type = "UNKNOWN"; int verbosity = dump_opt['d']; int print_header = 1; @@ -3414,7 +3414,8 @@ zdb_read_block(char *thing, spa_t *spa) psize = size; lsize = size; - pbuf = umem_alloc_aligned(SPA_MAXBLOCKSIZE, 512, UMEM_NOFAIL); + /* Some 4K native devices require 4K buffer alignment */ + pbuf = umem_alloc_aligned(SPA_MAXBLOCKSIZE, PAGESIZE, UMEM_NOFAIL); lbuf = umem_alloc(SPA_MAXBLOCKSIZE, UMEM_NOFAIL); BP_ZERO(bp); @@ -3481,9 +3482,18 @@ zdb_read_block(char *thing, spa_t *spa) VERIFY(random_get_pseudo_bytes((uint8_t *)pbuf2 + psize, SPA_MAXBLOCKSIZE - psize) == 0); - for (lsize = SPA_MAXBLOCKSIZE; lsize > psize; - lsize -= SPA_MINBLOCKSIZE) { + /* + * XXX - On the one hand, with SPA_MAXBLOCKSIZE at 16MB, + * this could take a while and we should let the user know + * we are not stuck. On the other hand, printing progress + * info gets old after a while. What to do? + */ + for (lsize = psize + SPA_MINBLOCKSIZE; + lsize <= SPA_MAXBLOCKSIZE; lsize += SPA_MINBLOCKSIZE) { for (c = 0; c < ZIO_COMPRESS_FUNCTIONS; c++) { + (void) printf("Trying %05llx -> %05llx (%s)\n", + (u_longlong_t)psize, (u_longlong_t)lsize, + zio_compress_table[c].ci_name); if (zio_decompress_data(c, pbuf, lbuf, psize, lsize) == 0 && zio_decompress_data(c, pbuf2, lbuf2, @@ -3552,7 +3562,7 @@ find_zpool(char **target, nvlist_t **configp, int dirc, char **dirv) nvlist_t *match = NULL; char *name = NULL; char *sepp = NULL; - char sep = 0; + char sep = '\0'; int count = 0; importargs_t args = { 0 }; diff --git a/cmd/zdb/zdb_il.c b/cmd/zdb/zdb_il.c index 93b905793d60..1501e879d207 100644 --- a/cmd/zdb/zdb_il.c +++ b/cmd/zdb/zdb_il.c @@ -80,8 +80,10 @@ zil_prt_rec_create(zilog_t *zilog, int txtype, lr_create_t *lr) } (void) printf("%s%s", prefix, ctime(&crtime)); - (void) printf("%sdoid %llu, foid %llu, mode %llo\n", prefix, - (u_longlong_t)lr->lr_doid, (u_longlong_t)lr->lr_foid, + (void) printf("%sdoid %llu, foid %llu, slots %llu, mode %llo\n", prefix, + (u_longlong_t)lr->lr_doid, + (u_longlong_t)LR_FOID_GET_OBJ(lr->lr_foid), + (u_longlong_t)LR_FOID_GET_SLOTS(lr->lr_foid), (longlong_t)lr->lr_mode); (void) printf("%suid %llu, gid %llu, gen %llu, rdev 0x%llx\n", prefix, (u_longlong_t)lr->lr_uid, (u_longlong_t)lr->lr_gid, diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c index 7525afcbfd6d..8a8f56af9928 100644 --- a/cmd/zfs/zfs_main.c +++ b/cmd/zfs/zfs_main.c @@ -248,10 +248,11 @@ get_usage(zfs_help_t idx) case HELP_PROMOTE: return (gettext("\tpromote \n")); case HELP_RECEIVE: - return (gettext("\treceive [-vnFu] \n" - "\treceive [-vnFu] [-o origin=] [-d | -e] " - "\n")); + "\treceive [-vnsFu] [-o origin=] [-d | -e] " + "\n" + "\treceive -A \n")); case HELP_RENAME: return (gettext("\trename [-f] " "\n" @@ -263,7 +264,8 @@ get_usage(zfs_help_t idx) return (gettext("\tsend [-DnPpRvLe] [-[iI] snapshot] " "\n" "\tsend [-Le] [-i snapshot|bookmark] " - "\n")); + "\n" + "\tsend [-nvPe] -t \n")); case HELP_SET: return (gettext("\tset ... " " ...\n")); @@ -610,7 +612,12 @@ zfs_mount_and_share(libzfs_handle_t *hdl, const char *dataset, zfs_type_t type) */ if (zfs_prop_valid_for_type(ZFS_PROP_CANMOUNT, type, B_FALSE) && zfs_prop_get_int(zhp, ZFS_PROP_CANMOUNT) == ZFS_CANMOUNT_ON) { - if (zfs_mount(zhp, NULL, 0) != 0) { + if (geteuid() != 0) { + (void) fprintf(stderr, gettext("filesystem " + "successfully created, but it may only be " + "mounted by root\n")); + ret = 1; + } else if (zfs_mount(zhp, NULL, 0) != 0) { (void) fprintf(stderr, gettext("filesystem " "successfully created, but not mounted\n")); ret = 1; @@ -1476,7 +1483,7 @@ get_callback(zfs_handle_t *zhp, void *data) char buf[ZFS_MAXPROPLEN]; char rbuf[ZFS_MAXPROPLEN]; zprop_source_t sourcetype; - char source[ZFS_MAXNAMELEN]; + char source[ZFS_MAX_DATASET_NAME_LEN]; zprop_get_cbdata_t *cbp = data; nvlist_t *user_props = zfs_get_user_props(zhp); zprop_list_t *pl = cbp->cb_proplist; @@ -1956,7 +1963,7 @@ typedef struct upgrade_cbdata { uint64_t cb_version; boolean_t cb_newer; boolean_t cb_foundone; - char cb_lastfs[ZFS_MAXNAMELEN]; + char cb_lastfs[ZFS_MAX_DATASET_NAME_LEN]; } upgrade_cbdata_t; static int @@ -2405,7 +2412,7 @@ userspace_cb(void *arg, const char *domain, uid_t rid, uint64_t space) if (domain != NULL && domain[0] != '\0') { #ifdef HAVE_IDMAP /* SMB */ - char sid[ZFS_MAXNAMELEN + 32]; + char sid[MAXNAMELEN + 32]; uid_t id; uint64_t classes; int err; @@ -2539,7 +2546,7 @@ print_us_node(boolean_t scripted, boolean_t parsable, int *fields, int types, size_t *width, us_node_t *node) { nvlist_t *nvl = node->usn_nvl; - char valstr[ZFS_MAXNAMELEN]; + char valstr[MAXNAMELEN]; boolean_t first = B_TRUE; int cfield = 0; int field; @@ -3410,7 +3417,7 @@ zfs_do_rollback(int argc, char **argv) boolean_t force = B_FALSE; rollback_cbdata_t cb = { 0 }; zfs_handle_t *zhp, *snap; - char parentname[ZFS_MAXNAMELEN]; + char parentname[ZFS_MAX_DATASET_NAME_LEN]; char *delim; /* check options */ @@ -3702,6 +3709,7 @@ zfs_do_send(int argc, char **argv) { char *fromname = NULL; char *toname = NULL; + char *resume_token = NULL; char *cp; zfs_handle_t *zhp; sendflags_t flags = { 0 }; @@ -3710,7 +3718,7 @@ zfs_do_send(int argc, char **argv) boolean_t extraverbose = B_FALSE; /* check options */ - while ((c = getopt(argc, argv, ":i:I:RDpvnPLe")) != -1) { + while ((c = getopt(argc, argv, ":i:I:RDpvnPLet:")) != -1) { switch (c) { case 'i': if (fromname) @@ -3751,6 +3759,9 @@ zfs_do_send(int argc, char **argv) case 'e': flags.embed_data = B_TRUE; break; + case 't': + resume_token = optarg; + break; case ':': (void) fprintf(stderr, gettext("missing argument for " "'%c' option\n"), optopt); @@ -3766,14 +3777,28 @@ zfs_do_send(int argc, char **argv) argc -= optind; argv += optind; - /* check number of arguments */ - if (argc < 1) { - (void) fprintf(stderr, gettext("missing snapshot argument\n")); - usage(B_FALSE); - } - if (argc > 1) { - (void) fprintf(stderr, gettext("too many arguments\n")); - usage(B_FALSE); + if (resume_token != NULL) { + if (fromname != NULL || flags.replicate || flags.props || + flags.dedup) { + (void) fprintf(stderr, + gettext("invalid flags combined with -t\n")); + usage(B_FALSE); + } + if (argc != 0) { + (void) fprintf(stderr, gettext("no additional " + "arguments are permitted with -t\n")); + usage(B_FALSE); + } + } else { + if (argc < 1) { + (void) fprintf(stderr, + gettext("missing snapshot argument\n")); + usage(B_FALSE); + } + if (argc > 1) { + (void) fprintf(stderr, gettext("too many arguments\n")); + usage(B_FALSE); + } } if (!flags.dryrun && isatty(STDOUT_FILENO)) { @@ -3783,12 +3808,17 @@ zfs_do_send(int argc, char **argv) return (1); } + if (resume_token != NULL) { + return (zfs_send_resume(g_zfs, &flags, STDOUT_FILENO, + resume_token)); + } + /* * Special case sending a filesystem, or from a bookmark. */ if (strchr(argv[0], '@') == NULL || (fromname && strchr(fromname, '#') != NULL)) { - char frombuf[ZFS_MAXNAMELEN]; + char frombuf[ZFS_MAX_DATASET_NAME_LEN]; enum lzc_send_flags lzc_flags = 0; if (flags.replicate || flags.doall || flags.props || @@ -3840,7 +3870,7 @@ zfs_do_send(int argc, char **argv) * case if they specify the origin. */ if (fromname && (cp = strchr(fromname, '@')) != NULL) { - char origin[ZFS_MAXNAMELEN]; + char origin[ZFS_MAX_DATASET_NAME_LEN]; zprop_source_t src; (void) zfs_prop_get(zhp, ZFS_PROP_ORIGIN, @@ -3888,8 +3918,6 @@ zfs_do_send(int argc, char **argv) } /* - * zfs receive [-vnFu] [-d | -e] - * * Restore a backup stream from stdin. */ static int @@ -3897,6 +3925,8 @@ zfs_do_receive(int argc, char **argv) { int c, err; recvflags_t flags = { 0 }; + boolean_t abort_resumable = B_FALSE; + nvlist_t *props; nvpair_t *nvp = NULL; @@ -3904,7 +3934,7 @@ zfs_do_receive(int argc, char **argv) nomem(); /* check options */ - while ((c = getopt(argc, argv, ":o:denuvF")) != -1) { + while ((c = getopt(argc, argv, ":o:denuvFsA")) != -1) { switch (c) { case 'o': if (parseprop(props, optarg) != 0) @@ -3926,9 +3956,15 @@ zfs_do_receive(int argc, char **argv) case 'v': flags.verbose = B_TRUE; break; + case 's': + flags.resumable = B_TRUE; + break; case 'F': flags.force = B_TRUE; break; + case 'A': + abort_resumable = B_TRUE; + break; case ':': (void) fprintf(stderr, gettext("missing argument for " "'%c' option\n"), optopt); @@ -3961,6 +3997,44 @@ zfs_do_receive(int argc, char **argv) } } + if (abort_resumable) { + if (flags.isprefix || flags.istail || flags.dryrun || + flags.resumable || flags.nomount) { + (void) fprintf(stderr, gettext("invalid option")); + usage(B_FALSE); + } + + char namebuf[ZFS_MAX_DATASET_NAME_LEN]; + (void) snprintf(namebuf, sizeof (namebuf), + "%s/%%recv", argv[0]); + + if (zfs_dataset_exists(g_zfs, namebuf, + ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME)) { + zfs_handle_t *zhp = zfs_open(g_zfs, + namebuf, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME); + if (zhp == NULL) + return (1); + err = zfs_destroy(zhp, B_FALSE); + } else { + zfs_handle_t *zhp = zfs_open(g_zfs, + argv[0], ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME); + if (zhp == NULL) + usage(B_FALSE); + if (!zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) || + zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN, + NULL, 0, NULL, NULL, 0, B_TRUE) == -1) { + (void) fprintf(stderr, + gettext("'%s' does not have any " + "resumable receive state to abort\n"), + argv[0]); + return (1); + } + err = zfs_destroy(zhp, B_FALSE); + } + + return (err != 0); + } + if (isatty(STDIN_FILENO)) { (void) fprintf(stderr, gettext("Error: Backup stream can not be read " @@ -3968,7 +4042,6 @@ zfs_do_receive(int argc, char **argv) "You must redirect standard input.\n")); return (1); } - err = zfs_receive(g_zfs, argv[0], props, &flags, STDIN_FILENO, NULL); return (err != 0); @@ -4480,7 +4553,7 @@ parse_fs_perm(fs_perm_t *fsperm, nvlist_t *nvl) who_perm = &node->who_perm; } } - + VERIFY3P(who_perm, !=, NULL); (void) parse_who_perm(who_perm, nvl2, perm_locality); } @@ -4787,7 +4860,7 @@ store_allow_perm(zfs_deleg_who_type_t type, boolean_t local, boolean_t descend, { int i; char ld[2] = { '\0', '\0' }; - char who_buf[ZFS_MAXNAMELEN+32]; + char who_buf[MAXNAMELEN + 32]; char base_type = ZFS_DELEG_WHO_UNKNOWN; char set_type = ZFS_DELEG_WHO_UNKNOWN; nvlist_t *base_nvl = NULL; @@ -5151,7 +5224,7 @@ static void print_fs_perms(fs_perm_set_t *fspset) { fs_perm_node_t *node = NULL; - char buf[ZFS_MAXNAMELEN+32]; + char buf[MAXNAMELEN + 32]; const char *dsname = buf; for (node = uu_list_first(fspset->fsps_list); node != NULL; @@ -5160,7 +5233,7 @@ print_fs_perms(fs_perm_set_t *fspset) uu_avl_t *uge_avl = node->fspn_fsperm.fsp_uge_avl; int left = 0; - (void) snprintf(buf, ZFS_MAXNAMELEN+32, + (void) snprintf(buf, sizeof (buf), gettext("---- Permissions on %s "), node->fspn_fsperm.fsp_name); (void) printf("%s", dsname); @@ -5357,7 +5430,7 @@ zfs_do_hold_rele_impl(int argc, char **argv, boolean_t holding) for (i = 0; i < argc; ++i) { zfs_handle_t *zhp; - char parent[ZFS_MAXNAMELEN]; + char parent[ZFS_MAX_DATASET_NAME_LEN]; const char *delim; char *path = argv[i]; @@ -5485,7 +5558,7 @@ holds_callback(zfs_handle_t *zhp, void *data) nvlist_t *nvl = NULL; nvpair_t *nvp = NULL; const char *zname = zfs_get_name(zhp); - size_t znamelen = strnlen(zname, ZFS_MAXNAMELEN); + size_t znamelen = strlen(zname); if (cbp->cb_recursive) { const char *snapname; @@ -5506,7 +5579,7 @@ holds_callback(zfs_handle_t *zhp, void *data) while ((nvp = nvlist_next_nvpair(nvl, nvp)) != NULL) { const char *tag = nvpair_name(nvp); - size_t taglen = strnlen(tag, MAXNAMELEN); + size_t taglen = strlen(tag); if (taglen > cbp->cb_max_taglen) cbp->cb_max_taglen = taglen; } @@ -5798,6 +5871,24 @@ share_mount_one(zfs_handle_t *zhp, int op, int flags, char *protocol, return (0); } + /* + * If this filesystem is inconsistent and has a receive resume + * token, we can not mount it. + */ + if (zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) && + zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN, + NULL, 0, NULL, NULL, 0, B_TRUE) == 0) { + if (!explicit) + return (0); + + (void) fprintf(stderr, gettext("cannot %s '%s': " + "Contains partially-completed state from " + "\"zfs receive -r\", which can be resumed with " + "\"zfs send -t\"\n"), + cmdname, zfs_get_name(zhp)); + return (1); + } + /* * At this point, we have verified that the mountpoint and/or * shareopts are appropriate for auto management. If the @@ -6604,7 +6695,7 @@ zfs_do_diff(int argc, char **argv) static int zfs_do_bookmark(int argc, char **argv) { - char snapname[ZFS_MAXNAMELEN]; + char snapname[ZFS_MAX_DATASET_NAME_LEN]; zfs_handle_t *zhp; nvlist_t *nvl; int ret = 0; diff --git a/cmd/zhack/zhack.c b/cmd/zhack/zhack.c index eedd17c30710..cc4f174c661e 100644 --- a/cmd/zhack/zhack.c +++ b/cmd/zhack/zhack.c @@ -48,7 +48,6 @@ #include #include #include -#undef ZFS_MAXNAMELEN #include extern boolean_t zfeature_checks_disable; diff --git a/cmd/zinject/zinject.c b/cmd/zinject/zinject.c index f64e504627a8..7e8381325e57 100644 --- a/cmd/zinject/zinject.c +++ b/cmd/zinject/zinject.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012 by Delphix. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. */ /* @@ -222,7 +222,7 @@ usage(void) "\tzinject -c \n" "\n" "\t\tClear the particular record (if given a numeric ID), or\n" - "\t\tall records if 'all' is specificed.\n" + "\t\tall records if 'all' is specified.\n" "\n" "\tzinject -p pool\n" "\t\tInject a panic fault at the specified function. Only \n" @@ -239,6 +239,38 @@ usage(void) "\tzinject -d device -A -D pool\n" "\t\tPerform a specific action on a particular device.\n" "\n" + "\tzinject -d device -D latency:lanes pool\n" + "\n" + "\t\tAdd an artificial delay to IO requests on a particular\n" + "\t\tdevice, such that the requests take a minimum of 'latency'\n" + "\t\tmilliseconds to complete. Each delay has an associated\n" + "\t\tnumber of 'lanes' which defines the number of concurrent\n" + "\t\tIO requests that can be processed.\n" + "\n" + "\t\tFor example, with a single lane delay of 10 ms (-D 10:1),\n" + "\t\tthe device will only be able to service a single IO request\n" + "\t\tat a time with each request taking 10 ms to complete. So,\n" + "\t\tif only a single request is submitted every 10 ms, the\n" + "\t\taverage latency will be 10 ms; but if more than one request\n" + "\t\tis submitted every 10 ms, the average latency will be more\n" + "\t\tthan 10 ms.\n" + "\n" + "\t\tSimilarly, if a delay of 10 ms is specified to have two\n" + "\t\tlanes (-D 10:2), then the device will be able to service\n" + "\t\ttwo requests at a time, each with a minimum latency of\n" + "\t\t10 ms. So, if two requests are submitted every 10 ms, then\n" + "\t\tthe average latency will be 10 ms; but if more than two\n" + "\t\trequests are submitted every 10 ms, the average latency\n" + "\t\twill be more than 10 ms.\n" + "\n" + "\t\tAlso note, these delays are additive. So two invocations\n" + "\t\tof '-D 10:1', is roughly equivalent to a single invocation\n" + "\t\tof '-D 10:2'. This also means, one can specify multiple\n" + "\t\tlanes with differing target latencies. For example, an\n" + "\t\tinvocation of '-D 10:1' followed by '-D 25:2' will\n" + "\t\tcreate 3 lanes on the device; one lane with a latency\n" + "\t\tof 10 ms and two lanes with a 25 ms latency.\n" + "\n" "\tzinject -I [-s | -g ] pool\n" "\t\tCause the pool to stop writing blocks yet not\n" "\t\treport errors for a duration. Simulates buggy hardware\n" @@ -257,7 +289,7 @@ usage(void) "\n" "\t\tInject an error into the object specified by the '-t' option\n" "\t\tand the object descriptor. The 'object' parameter is\n" - "\t\tinterperted depending on the '-t' option.\n" + "\t\tinterpreted depending on the '-t' option.\n" "\n" "\t\t-q\tQuiet mode. Only print out the handler number added.\n" "\t\t-e\tInject a specific error. Must be either 'io' or\n" @@ -353,6 +385,9 @@ print_device_handler(int id, const char *pool, zinject_record_t *record, if (record->zi_guid == 0 || record->zi_func[0] != '\0') return (0); + if (record->zi_cmd == ZINJECT_DELAY_IO) + return (0); + if (*count == 0) { (void) printf("%3s %-15s %s\n", "ID", "POOL", "GUID"); (void) printf("--- --------------- ----------------\n"); @@ -366,6 +401,35 @@ print_device_handler(int id, const char *pool, zinject_record_t *record, return (0); } +static int +print_delay_handler(int id, const char *pool, zinject_record_t *record, + void *data) +{ + int *count = data; + + if (record->zi_guid == 0 || record->zi_func[0] != '\0') + return (0); + + if (record->zi_cmd != ZINJECT_DELAY_IO) + return (0); + + if (*count == 0) { + (void) printf("%3s %-15s %-15s %-15s %s\n", + "ID", "POOL", "DELAY (ms)", "LANES", "GUID"); + (void) printf("--- --------------- --------------- " + "--------------- ----------------\n"); + } + + *count += 1; + + (void) printf("%3d %-15s %-15llu %-15llu %llx\n", id, pool, + (u_longlong_t)NSEC2MSEC(record->zi_timer), + (u_longlong_t)record->zi_nlanes, + (u_longlong_t)record->zi_guid); + + return (0); +} + static int print_panic_handler(int id, const char *pool, zinject_record_t *record, void *data) @@ -403,6 +467,13 @@ print_all_handlers(void) count = 0; } + (void) iter_handlers(print_delay_handler, &count); + if (count > 0) { + total += count; + (void) printf("\n"); + count = 0; + } + (void) iter_handlers(print_data_handler, &count); if (count > 0) { total += count; @@ -545,6 +616,35 @@ perform_action(const char *pool, zinject_record_t *record, int cmd) return (1); } +static int +parse_delay(char *str, uint64_t *delay, uint64_t *nlanes) +{ + unsigned long scan_delay; + unsigned long scan_nlanes; + + if (sscanf(str, "%lu:%lu", &scan_delay, &scan_nlanes) != 2) + return (1); + + /* + * We explicitly disallow a delay of zero here, because we key + * off this value being non-zero in translate_device(), to + * determine if the fault is a ZINJECT_DELAY_IO fault or not. + */ + if (scan_delay == 0) + return (1); + + /* + * The units for the CLI delay parameter is milliseconds, but + * the data passed to the kernel is interpreted as nanoseconds. + * Thus we scale the milliseconds to nanoseconds here, and this + * nanosecond value is used to pass the delay to the kernel. + */ + *delay = MSEC2NSEC(scan_delay); + *nlanes = scan_nlanes; + + return (0); +} + int main(int argc, char **argv) { @@ -563,8 +663,8 @@ main(int argc, char **argv) err_type_t type = TYPE_INVAL; err_type_t label = TYPE_INVAL; zinject_record_t record = { 0 }; - char pool[MAXNAMELEN]; - char dataset[MAXNAMELEN]; + char pool[MAXNAMELEN] = ""; + char dataset[MAXNAMELEN] = ""; zfs_handle_t *zhp = NULL; int nowrites = 0; int dur_txg = 0; @@ -628,8 +728,10 @@ main(int argc, char **argv) break; case 'D': errno = 0; - record.zi_timer = strtoull(optarg, &end, 10); - if (errno != 0 || *end != '\0') { + ret = parse_delay(optarg, &record.zi_timer, + &record.zi_nlanes); + if (ret != 0) { + (void) fprintf(stderr, "invalid i/o delay " "value: '%s'\n", optarg); usage(); diff --git a/cmd/zpios/zpios_main.c b/cmd/zpios/zpios_main.c index e6e88f60d940..d0e3f7b91aac 100644 --- a/cmd/zpios/zpios_main.c +++ b/cmd/zpios/zpios_main.c @@ -343,7 +343,7 @@ args_init(int argc, char **argv) check_mutual_exclusive_command_lines(fl_cs, "chunksize"); if (args->pool == NULL) { - fprintf(stderr, "Error: Pool not specificed\n"); + fprintf(stderr, "Error: Pool not specified\n"); usage(); args_fini(args); return (NULL); diff --git a/cmd/zpool/zpool_iter.c b/cmd/zpool/zpool_iter.c index a18ccf29df33..1b64a5a5c9f2 100644 --- a/cmd/zpool/zpool_iter.c +++ b/cmd/zpool/zpool_iter.c @@ -308,7 +308,7 @@ for_each_vdev_cb(zpool_handle_t *zhp, nvlist_t *nv, pool_vdev_iter_f func, int for_each_vdev(zpool_handle_t *zhp, pool_vdev_iter_f func, void *data) { - nvlist_t *config, *nvroot; + nvlist_t *config, *nvroot = NULL; if ((config = zpool_get_config(zhp, NULL)) != NULL) { verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, diff --git a/cmd/zpool/zpool_main.c b/cmd/zpool/zpool_main.c index 414ed69c57d0..9041f9c33e15 100644 --- a/cmd/zpool/zpool_main.c +++ b/cmd/zpool/zpool_main.c @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include @@ -153,6 +152,7 @@ enum iostat_type { IOS_LATENCY = 1, IOS_QUEUES = 2, IOS_L_HISTO = 3, + IOS_RQ_HISTO = 4, IOS_COUNT, /* always last element */ }; @@ -161,6 +161,62 @@ enum iostat_type { #define IOS_LATENCY_M (1ULL << IOS_LATENCY) #define IOS_QUEUES_M (1ULL << IOS_QUEUES) #define IOS_L_HISTO_M (1ULL << IOS_L_HISTO) +#define IOS_RQ_HISTO_M (1ULL << IOS_RQ_HISTO) + +/* Mask of all the histo bits */ +#define IOS_ANYHISTO_M (IOS_L_HISTO_M | IOS_RQ_HISTO_M) + +/* + * Lookup table for iostat flags to nvlist names. Basically a list + * of all the nvlists a flag requires. Also specifies the order in + * which data gets printed in zpool iostat. + */ +static const char *vsx_type_to_nvlist[IOS_COUNT][11] = { + [IOS_L_HISTO] = { + ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO, + ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO, + ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO, + ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO, + ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO, + ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO, + ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO, + ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO, + ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO, + NULL}, + [IOS_LATENCY] = { + ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO, + ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO, + ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO, + ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO, + NULL}, + [IOS_QUEUES] = { + ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE, + ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE, + ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE, + ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE, + ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE, + NULL}, + [IOS_RQ_HISTO] = { + ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO, + ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO, + ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO, + ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO, + ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO, + ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO, + ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO, + ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO, + ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO, + ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO, + NULL}, +}; + + +/* + * Given a cb->cb_flags with a histogram bit set, return the iostat_type. + * Right now, only one histo bit is ever set at one time, so we can + * just do a highbit64(a) + */ +#define IOS_HISTO_IDX(a) (highbit64(a & IOS_ANYHISTO_M) - 1) typedef struct zpool_command { const char *name; @@ -255,7 +311,8 @@ get_usage(zpool_help_t idx) { "[-R root] [-F [-n]]\n" "\t [newpool]\n")); case HELP_IOSTAT: - return (gettext("\tiostat [-T d | u] [-ghHLpPvy] [[-lq]|-w]\n" + return (gettext("\tiostat [-T d | u] [-ghHLpPvy] " + "[[-lq]|[-r|-w]]\n" "\t [[pool ...]|[pool vdev ...]|[vdev ...]] " "[interval [count]]\n")); case HELP_LABELCLEAR: @@ -2273,21 +2330,20 @@ zpool_do_import(int argc, char **argv) (void) fprintf(stderr, gettext("too many arguments\n")); usage(B_FALSE); } + } - /* - * Check for the SYS_CONFIG privilege. We do this explicitly - * here because otherwise any attempt to discover pools will - * silently fail. - */ - if (argc == 0 && !priv_ineffect(PRIV_SYS_CONFIG)) { - (void) fprintf(stderr, gettext("cannot " - "discover pools: permission denied\n")); - if (searchdirs != NULL) - free(searchdirs); + /* + * Check for the effective uid. We do this explicitly here because + * otherwise any attempt to discover pools will silently fail. + */ + if (argc == 0 && geteuid() != 0) { + (void) fprintf(stderr, gettext("cannot " + "discover pools: permission denied\n")); + if (searchdirs != NULL) + free(searchdirs); - nvlist_free(policy); - return (1); - } + nvlist_free(policy); + return (1); } /* @@ -2531,6 +2587,9 @@ static const name_and_columns_t iostat_top_labels[][IOSTAT_MAX_LABELS] = {NULL}}, [IOS_L_HISTO] = {{"total_wait", 2}, {"disk_wait", 2}, {"sync_queue", 2}, {"async_queue", 2}, {NULL}}, + [IOS_RQ_HISTO] = {{"sync_read", 2}, {"sync_write", 2}, + {"async_read", 2}, {"async_write", 2}, {"scrub", 2}, {NULL}}, + }; /* Shorthand - if "columns" field not set, default to 1 column */ @@ -2544,6 +2603,13 @@ static const name_and_columns_t iostat_bottom_labels[][IOSTAT_MAX_LABELS] = {"activ"}, {"pend"}, {"activ"}, {"pend"}, {"activ"}, {NULL}}, [IOS_L_HISTO] = {{"read"}, {"write"}, {"read"}, {"write"}, {"read"}, {"write"}, {"read"}, {"write"}, {"scrub"}, {NULL}}, + [IOS_RQ_HISTO] = {{"ind"}, {"agg"}, {"ind"}, {"agg"}, {"ind"}, {"agg"}, + {"ind"}, {"agg"}, {"ind"}, {"agg"}, {NULL}}, +}; + +static const char *histo_to_title[] = { + [IOS_L_HISTO] = "latency", + [IOS_RQ_HISTO] = "req_size", }; /* @@ -2562,6 +2628,25 @@ label_array_len(const name_and_columns_t *labels) return (i); } +/* + * Return the number of strings in a null-terminated string array. + * For example: + * + * const char foo[] = {"bar", "baz", NULL} + * + * returns 2 + */ +static uint64_t +str_array_len(const char *array[]) +{ + uint64_t i = 0; + while (array[i]) + i++; + + return (i); +} + + /* * Return a default column width for default/latency/queue columns. This does * not include histograms, which have their columns autosized. @@ -2673,14 +2758,22 @@ print_iostat_dashes(iostat_cbdata_t *cb, unsigned int force_column_width, uint64_t f; int idx; const name_and_columns_t *labels; + const char *title; + + + if (cb->cb_flags & IOS_ANYHISTO_M) { + title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]; + } else if (cb->cb_vdev_names_count) { + title = "vdev"; + } else { + title = "pool"; + } + + namewidth = MAX(MAX(strlen(title), cb->cb_namewidth), + name ? strlen(name) : 0); - if (cb->cb_flags & IOS_L_HISTO_M) - namewidth = MAX(cb->cb_namewidth, strlen("latency")); - else - namewidth = cb->cb_namewidth; if (name) { - namewidth = MAX(cb->cb_namewidth, strlen(name)); printf("%-*s", namewidth, name); } else { for (i = 0; i < namewidth; i++) @@ -2727,22 +2820,28 @@ print_iostat_header_impl(iostat_cbdata_t *cb, unsigned int force_column_width, const char *histo_vdev_name) { unsigned int namewidth; - uint64_t flags = cb->cb_flags; + const char *title; - if (flags & IOS_L_HISTO_M) - namewidth = MAX(cb->cb_namewidth, strlen("latency")); - else - namewidth = cb->cb_namewidth; + if (cb->cb_flags & IOS_ANYHISTO_M) { + title = histo_to_title[IOS_HISTO_IDX(cb->cb_flags)]; + } else if (cb->cb_vdev_names_count) { + title = "vdev"; + } else { + title = "pool"; + } + + namewidth = MAX(MAX(strlen(title), cb->cb_namewidth), + histo_vdev_name ? strlen(histo_vdev_name) : 0); - if (flags & IOS_L_HISTO_M) + if (histo_vdev_name) printf("%-*s", namewidth, histo_vdev_name); else printf("%*s", namewidth, ""); + print_iostat_labels(cb, force_column_width, iostat_top_labels); - printf("%-*s", namewidth, flags & IOS_L_HISTO_M ? "latency" : - cb->cb_vdev_names_count ? "vdev" : "pool"); + printf("%-*s", namewidth, title); print_iostat_labels(cb, force_column_width, iostat_bottom_labels); @@ -2918,6 +3017,7 @@ print_iostat_histo(struct stat_array *nva, unsigned int len, uint64_t val; enum zfs_nicenum_format format; unsigned int buckets; + unsigned int start_bucket; if (cb->cb_literal) format = ZFS_NICENUM_RAW; @@ -2927,12 +3027,25 @@ print_iostat_histo(struct stat_array *nva, unsigned int len, /* All these histos are the same size, so just use nva[0].count */ buckets = nva[0].count; - for (j = 0; j < buckets; j++) { - /* Ending range of this bucket */ - val = (1UL << (j + 1)) - 1; + if (cb->cb_flags & IOS_RQ_HISTO_M) { + /* Start at 512 - req size should never be lower than this */ + start_bucket = 9; + } else { + start_bucket = 0; + } + for (j = start_bucket; j < buckets; j++) { /* Print histogram bucket label */ - zfs_nicetime(val, buf, sizeof (buf)); + if (cb->cb_flags & IOS_L_HISTO_M) { + /* Ending range of this bucket */ + val = (1UL << (j + 1)) - 1; + zfs_nicetime(val, buf, sizeof (buf)); + } else { + /* Request size (starting range of bucket) */ + val = (1UL << j); + zfs_nicenum(val, buf, sizeof (buf)); + } + if (cb->cb_scripted) printf("%llu", (u_longlong_t) val); else @@ -2962,30 +3075,29 @@ print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv, unsigned int column_width; unsigned int namewidth; unsigned int entire_width; - - const char *names[] = { - ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO, - ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO, - ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO, - ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO, - ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO, - }; + enum iostat_type type; struct stat_array *nva; - nva = calc_and_alloc_stats_ex(names, ARRAY_SIZE(names), oldnv, newnv); + const char **names; + unsigned int names_len; + + /* What type of histo are we? */ + type = IOS_HISTO_IDX(cb->cb_flags); + + /* Get NULL-terminated array of nvlist names for our histo */ + names = vsx_type_to_nvlist[type]; + names_len = str_array_len(names); /* num of names */ + + nva = calc_and_alloc_stats_ex(names, names_len, oldnv, newnv); if (cb->cb_literal) { column_width = MAX(5, - (unsigned int) log10(stat_histo_max(nva, - ARRAY_SIZE(names))) + 1); + (unsigned int) log10(stat_histo_max(nva, names_len)) + 1); } else { column_width = 5; } - namewidth = MAX(cb->cb_namewidth, strlen("latency")); + namewidth = MAX(cb->cb_namewidth, + strlen(histo_to_title[IOS_HISTO_IDX(cb->cb_flags)])); /* * Calculate the entire line width of what we're printing. The @@ -2998,17 +3110,17 @@ print_iostat_histos(iostat_cbdata_t *cb, nvlist_t *oldnv, /* |__________| <--- entire_width */ /* */ entire_width = namewidth + (column_width + 2) * - label_array_len(iostat_bottom_labels[IOS_L_HISTO]); + label_array_len(iostat_bottom_labels[type]); if (cb->cb_scripted) printf("%s\n", name); else print_iostat_header_impl(cb, column_width, name); - print_iostat_histo(nva, ARRAY_SIZE(names), cb, column_width, + print_iostat_histo(nva, names_len, cb, column_width, namewidth, scale); - free_calc_stats(nva, ARRAY_SIZE(names)); + free_calc_stats(nva, names_len); if (!cb->cb_scripted) print_solid_separator(entire_width); } @@ -3219,7 +3331,7 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv, * Print the vdev name unless it's is a histogram. Histograms * display the vdev name in the header itself. */ - if (!(cb->cb_flags & IOS_L_HISTO_M)) { + if (!(cb->cb_flags & IOS_ANYHISTO_M)) { if (cb->cb_scripted) { printf("%s", name); } else { @@ -3234,7 +3346,7 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv, /* Calculate our scaling factor */ tdelta = newvs->vs_timestamp - oldvs->vs_timestamp; - if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_L_HISTO_M)) { + if ((oldvs->vs_timestamp == 0) && (cb->cb_flags & IOS_ANYHISTO_M)) { /* * If we specify printing histograms with no time interval, then * print the histogram numbers over the entire lifetime of the @@ -3256,12 +3368,12 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv, print_iostat_latency(cb, oldnv, newnv, scale); if (cb->cb_flags & IOS_QUEUES_M) print_iostat_queues(cb, oldnv, newnv, scale); - if (cb->cb_flags & IOS_L_HISTO_M) { + if (cb->cb_flags & IOS_ANYHISTO_M) { printf("\n"); print_iostat_histos(cb, oldnv, newnv, scale, name); } - if (!(cb->cb_flags & IOS_L_HISTO_M)) + if (!(cb->cb_flags & IOS_ANYHISTO_M)) printf("\n"); free(calcvs); @@ -3303,7 +3415,7 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv, */ if (num_logs(newnv) > 0) { - if ((!(cb->cb_flags & IOS_L_HISTO_M)) && !cb->cb_scripted && + if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted && !cb->cb_vdev_names) { print_iostat_dashes(cb, 0, "logs"); } @@ -3337,7 +3449,7 @@ print_vdev_stats(zpool_handle_t *zhp, const char *name, nvlist_t *oldnv, return (ret); if (children > 0) { - if ((!(cb->cb_flags & IOS_L_HISTO_M)) && !cb->cb_scripted && + if ((!(cb->cb_flags & IOS_ANYHISTO_M)) && !cb->cb_scripted && !cb->cb_vdev_names) { print_iostat_dashes(cb, 0, "cache"); } @@ -3399,9 +3511,10 @@ print_iostat(zpool_handle_t *zhp, void *data) ret = print_vdev_stats(zhp, zpool_get_name(zhp), oldnvroot, newnvroot, cb, 0); - if ((ret != 0) && !(cb->cb_flags & IOS_L_HISTO_M) && !cb->cb_scripted && - cb->cb_verbose && !cb->cb_vdev_names_count) - print_iostat_separator(cb); + if ((ret != 0) && !(cb->cb_flags & IOS_ANYHISTO_M) && + !cb->cb_scripted && cb->cb_verbose && !cb->cb_vdev_names_count) { + print_iostat_separator(cb); + } return (ret); } @@ -3553,37 +3666,6 @@ get_stat_flags_cb(zpool_handle_t *zhp, void *data) uint64_t flags = 0; int i, j; - /* - * Lookup table for extended iostat flags to nvlist names. - * Basically a list of all the nvpairs a flag requires. - */ - static const char *vsx_type_to_nvlist[IOS_COUNT][10] = { - [IOS_L_HISTO] = { - ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO, - ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO, - ZPOOL_CONFIG_VDEV_SYNC_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_SYNC_W_LAT_HISTO, - ZPOOL_CONFIG_VDEV_ASYNC_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO, - ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO, - NULL}, - [IOS_LATENCY] = { - ZPOOL_CONFIG_VDEV_TOT_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_TOT_W_LAT_HISTO, - ZPOOL_CONFIG_VDEV_DISK_R_LAT_HISTO, - ZPOOL_CONFIG_VDEV_DISK_W_LAT_HISTO, - NULL}, - [IOS_QUEUES] = { - ZPOOL_CONFIG_VDEV_SYNC_R_ACTIVE_QUEUE, - ZPOOL_CONFIG_VDEV_SYNC_W_ACTIVE_QUEUE, - ZPOOL_CONFIG_VDEV_ASYNC_R_ACTIVE_QUEUE, - ZPOOL_CONFIG_VDEV_ASYNC_W_ACTIVE_QUEUE, - ZPOOL_CONFIG_VDEV_SCRUB_ACTIVE_QUEUE, - NULL} - }; - config = zpool_get_config(zhp, NULL); verify(nvlist_lookup_nvlist(config, ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0); @@ -3818,7 +3900,7 @@ fsleep(float sec) { /* - * zpool iostat [-ghHLpPvy] [[-lq]-w] [-n name] [-T d|u] + * zpool iostat [-ghHLpPvy] [[-lq]|[-r|-w]] [-n name] [-T d|u] * [[ pool ...]|[pool vdev ...]|[vdev ...]] * [interval [count]] * @@ -3832,7 +3914,8 @@ fsleep(float sec) { * by a single tab. * -l Display average latency * -q Display queue depths - * -w Display histograms + * -w Display latency histograms + * -r Display request size histogram * -T Display a timestamp in date(1) or Unix format * * This command can be tricky because we want to be able to deal with pool @@ -3851,8 +3934,8 @@ zpool_do_iostat(int argc, char **argv) unsigned long count = 0; zpool_list_t *list; boolean_t verbose = B_FALSE; - boolean_t latency = B_FALSE, histo = B_FALSE; - boolean_t queues = B_FALSE, parseable = B_FALSE, scripted = B_FALSE; + boolean_t latency = B_FALSE, l_histo = B_FALSE, rq_histo = B_FALSE; + boolean_t queues = B_FALSE, parsable = B_FALSE, scripted = B_FALSE; boolean_t omit_since_boot = B_FALSE; boolean_t guid = B_FALSE; boolean_t follow_links = B_FALSE; @@ -3861,12 +3944,12 @@ zpool_do_iostat(int argc, char **argv) /* Used for printing error message */ const char flag_to_arg[] = {[IOS_LATENCY] = 'l', [IOS_QUEUES] = 'q', - [IOS_L_HISTO] = 'w'}; + [IOS_L_HISTO] = 'w', [IOS_RQ_HISTO] = 'r'}; uint64_t unsupported_flags; /* check options */ - while ((c = getopt(argc, argv, "gLPT:vyhplqwH")) != -1) { + while ((c = getopt(argc, argv, "gLPT:vyhplqrwH")) != -1) { switch (c) { case 'g': guid = B_TRUE; @@ -3884,7 +3967,7 @@ zpool_do_iostat(int argc, char **argv) verbose = B_TRUE; break; case 'p': - parseable = B_TRUE; + parsable = B_TRUE; break; case 'l': latency = B_TRUE; @@ -3896,7 +3979,10 @@ zpool_do_iostat(int argc, char **argv) scripted = B_TRUE; break; case 'w': - histo = B_TRUE; + l_histo = B_TRUE; + break; + case 'r': + rq_histo = B_TRUE; break; case 'y': omit_since_boot = B_TRUE; @@ -3914,7 +4000,7 @@ zpool_do_iostat(int argc, char **argv) argc -= optind; argv += optind; - cb.cb_literal = parseable; + cb.cb_literal = parsable; cb.cb_scripted = scripted; if (guid) @@ -3997,10 +4083,18 @@ zpool_do_iostat(int argc, char **argv) return (1); } - if (histo && (queues || latency)) { + if ((l_histo || rq_histo) && (queues || latency)) { + pool_list_free(list); + (void) fprintf(stderr, + gettext("[-r|-w] isn't allowed with [-q|-l]\n")); + usage(B_FALSE); + return (1); + } + + if (l_histo && rq_histo) { pool_list_free(list); (void) fprintf(stderr, - gettext("-w isn't allowed with [-q|-l]\n")); + gettext("Only one of [-r|-w] can be passed at a time\n")); usage(B_FALSE); return (1); } @@ -4010,13 +4104,15 @@ zpool_do_iostat(int argc, char **argv) */ cb.cb_list = list; - if (histo) { + if (l_histo) { /* * Histograms tables look out of place when you try to display * them with the other stats, so make a rule that you can only * print histograms by themselves. */ cb.cb_flags = IOS_L_HISTO_M; + } else if (rq_histo) { + cb.cb_flags = IOS_RQ_HISTO_M; } else { cb.cb_flags = IOS_DEFAULT_M; if (latency) @@ -4088,7 +4184,7 @@ zpool_do_iostat(int argc, char **argv) */ if (((++cb.cb_iteration == 1 && !skip) || (skip != verbose)) && - (!(cb.cb_flags & IOS_L_HISTO_M)) && + (!(cb.cb_flags & IOS_ANYHISTO_M)) && !cb.cb_scripted) print_iostat_header(&cb); @@ -4108,8 +4204,8 @@ zpool_do_iostat(int argc, char **argv) * we also want an ending separator. */ if (((npools > 1 && !verbose && - !(cb.cb_flags & IOS_L_HISTO_M)) || - (!(cb.cb_flags & IOS_L_HISTO_M) && + !(cb.cb_flags & IOS_ANYHISTO_M)) || + (!(cb.cb_flags & IOS_ANYHISTO_M) && cb.cb_vdev_names_count)) && !cb.cb_scripted) { print_iostat_separator(&cb); @@ -5762,7 +5858,7 @@ status_callback(zpool_handle_t *zhp, void *data) case ZPOOL_STATUS_BAD_LOG: (void) printf(gettext("status: An intent log record " "could not be read.\n" - "\tWaiting for adminstrator intervention to fix the " + "\tWaiting for administrator intervention to fix the " "faulted pool.\n")); (void) printf(gettext("action: Either restore the affected " "device(s) and run 'zpool online',\n" diff --git a/cmd/zpool/zpool_vdev.c b/cmd/zpool/zpool_vdev.c index 587aa7fd4861..7ad44a948c43 100644 --- a/cmd/zpool/zpool_vdev.c +++ b/cmd/zpool/zpool_vdev.c @@ -486,28 +486,19 @@ static int check_device(const char *path, boolean_t force, boolean_t isspare, boolean_t iswholedisk) { - static blkid_cache cache = NULL; - - /* - * There is no easy way to add a correct blkid_put_cache() call, - * memory will be reclaimed when the command exits. - */ - if (cache == NULL) { - int err; - - if ((err = blkid_get_cache(&cache, NULL)) != 0) { - check_error(err); - return (-1); - } + blkid_cache cache; + int error; - if ((err = blkid_probe_all(cache)) != 0) { - blkid_put_cache(cache); - check_error(err); - return (-1); - } + error = blkid_get_cache(&cache, NULL); + if (error != 0) { + check_error(error); + return (-1); } - return (check_disk(path, cache, force, isspare, iswholedisk)); + error = check_disk(path, cache, force, isspare, iswholedisk); + blkid_put_cache(cache); + + return (error); } /* @@ -1522,8 +1513,13 @@ construct_spec(nvlist_t *props, int argc, char **argv) if (child == NULL) zpool_no_memory(); if ((nv = make_leaf_vdev(props, argv[c], - B_FALSE)) == NULL) + B_FALSE)) == NULL) { + for (c = 0; c < children - 1; c++) + nvlist_free(child[c]); + free(child); return (NULL); + } + child[children - 1] = nv; } @@ -1531,6 +1527,9 @@ construct_spec(nvlist_t *props, int argc, char **argv) (void) fprintf(stderr, gettext("invalid vdev " "specification: %s requires at least %d " "devices\n"), argv[0], mindev); + for (c = 0; c < children; c++) + nvlist_free(child[c]); + free(child); return (NULL); } @@ -1538,6 +1537,9 @@ construct_spec(nvlist_t *props, int argc, char **argv) (void) fprintf(stderr, gettext("invalid vdev " "specification: %s supports no more than " "%d devices\n"), argv[0], maxdev); + for (c = 0; c < children; c++) + nvlist_free(child[c]); + free(child); return (NULL); } diff --git a/cmd/zstreamdump/zstreamdump.c b/cmd/zstreamdump/zstreamdump.c index f288d148e574..08d52bb37a83 100644 --- a/cmd/zstreamdump/zstreamdump.c +++ b/cmd/zstreamdump/zstreamdump.c @@ -127,7 +127,7 @@ read_hdr(dmu_replay_record_t *drr, zio_cksum_t *cksum) (longlong_t)saved_cksum.zc_word[1], (longlong_t)saved_cksum.zc_word[2], (longlong_t)saved_cksum.zc_word[3]); - exit(1); + return (0); } return (sizeof (*drr)); } @@ -347,8 +347,7 @@ main(int argc, char *argv[]) if (verbose) (void) printf("\n"); - if ((DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == - DMU_COMPOUNDSTREAM) && drr->drr_payloadlen != 0) { + if (drr->drr_payloadlen != 0) { nvlist_t *nv; int sz = drr->drr_payloadlen; diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c index bbf5cbfb78bf..852bf00a616b 100644 --- a/cmd/ztest/ztest.c +++ b/cmd/ztest/ztest.c @@ -123,6 +123,7 @@ #include #include #include +#include #include #ifdef __GLIBC__ #include /* for backtrace() */ @@ -144,8 +145,8 @@ typedef struct ztest_shared_hdr { static ztest_shared_hdr_t *ztest_shared_hdr; typedef struct ztest_shared_opts { - char zo_pool[MAXNAMELEN]; - char zo_dir[MAXNAMELEN]; + char zo_pool[ZFS_MAX_DATASET_NAME_LEN]; + char zo_dir[ZFS_MAX_DATASET_NAME_LEN]; char zo_alt_ztest[MAXNAMELEN]; char zo_alt_libpath[MAXNAMELEN]; uint64_t zo_vdevs; @@ -220,6 +221,7 @@ typedef struct ztest_block_tag { uint64_t bt_magic; uint64_t bt_objset; uint64_t bt_object; + uint64_t bt_dnodesize; uint64_t bt_offset; uint64_t bt_gen; uint64_t bt_txg; @@ -257,9 +259,10 @@ typedef struct ztest_od { dmu_object_type_t od_crtype; uint64_t od_blocksize; uint64_t od_crblocksize; + uint64_t od_crdnodesize; uint64_t od_gen; uint64_t od_crgen; - char od_name[MAXNAMELEN]; + char od_name[ZFS_MAX_DATASET_NAME_LEN]; } ztest_od_t; /* @@ -271,7 +274,7 @@ typedef struct ztest_ds { rwlock_t zd_zilog_lock; zilog_t *zd_zilog; ztest_od_t *zd_od; /* debugging aid */ - char zd_name[MAXNAMELEN]; + char zd_name[ZFS_MAX_DATASET_NAME_LEN]; kmutex_t zd_dirobj_lock; rll_t zd_object_lock[ZTEST_OBJECT_LOCKS]; zll_t zd_range_lock[ZTEST_RANGE_LOCKS]; @@ -327,6 +330,8 @@ ztest_func_t ztest_vdev_aux_add_remove; ztest_func_t ztest_split_pool; ztest_func_t ztest_reguid; ztest_func_t ztest_spa_upgrade; +ztest_func_t ztest_fletcher; +ztest_func_t ztest_verify_dnode_bt; uint64_t zopt_always = 0ULL * NANOSEC; /* all the time */ uint64_t zopt_incessant = 1ULL * NANOSEC / 10; /* every 1/10 second */ @@ -372,6 +377,8 @@ ztest_info_t ztest_info[] = { ZTI_INIT(ztest_vdev_LUN_growth, 1, &zopt_rarely), ZTI_INIT(ztest_vdev_add_remove, 1, &ztest_opts.zo_vdevtime), ZTI_INIT(ztest_vdev_aux_add_remove, 1, &ztest_opts.zo_vdevtime), + ZTI_INIT(ztest_fletcher, 1, &zopt_rarely), + ZTI_INIT(ztest_verify_dnode_bt, 1, &zopt_sometimes), }; #define ZTEST_FUNCS (sizeof (ztest_info) / sizeof (ztest_info_t)) @@ -719,6 +726,7 @@ process_options(int argc, char **argv) } else { (void) strlcpy(zo->zo_dir, path, sizeof (zo->zo_dir)); + free(path); } break; case 'V': @@ -1024,6 +1032,36 @@ ztest_random_blocksize(void) return (1 << (SPA_MINBLOCKSHIFT + block_shift)); } +static int +ztest_random_dnodesize(void) +{ + int slots; + int max_slots = spa_maxdnodesize(ztest_spa) >> DNODE_SHIFT; + + if (max_slots == DNODE_MIN_SLOTS) + return (DNODE_MIN_SIZE); + + /* + * Weight the random distribution more heavily toward smaller + * dnode sizes since that is more likely to reflect real-world + * usage. + */ + ASSERT3U(max_slots, >, 4); + switch (ztest_random(10)) { + case 0: + slots = 5 + ztest_random(max_slots - 4); + break; + case 1 ... 4: + slots = 2 + ztest_random(3); + break; + default: + slots = 1; + break; + } + + return (slots << DNODE_SHIFT); +} + static int ztest_random_ibshift(void) { @@ -1414,11 +1452,13 @@ ztest_pattern_match(void *buf, uint64_t size, uint64_t value) static void ztest_bt_generate(ztest_block_tag_t *bt, objset_t *os, uint64_t object, - uint64_t offset, uint64_t gen, uint64_t txg, uint64_t crtxg) + uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg, + uint64_t crtxg) { bt->bt_magic = BT_MAGIC; bt->bt_objset = dmu_objset_id(os); bt->bt_object = object; + bt->bt_dnodesize = dnodesize; bt->bt_offset = offset; bt->bt_gen = gen; bt->bt_txg = txg; @@ -1427,11 +1467,13 @@ ztest_bt_generate(ztest_block_tag_t *bt, objset_t *os, uint64_t object, static void ztest_bt_verify(ztest_block_tag_t *bt, objset_t *os, uint64_t object, - uint64_t offset, uint64_t gen, uint64_t txg, uint64_t crtxg) + uint64_t dnodesize, uint64_t offset, uint64_t gen, uint64_t txg, + uint64_t crtxg) { ASSERT3U(bt->bt_magic, ==, BT_MAGIC); ASSERT3U(bt->bt_objset, ==, dmu_objset_id(os)); ASSERT3U(bt->bt_object, ==, object); + ASSERT3U(bt->bt_dnodesize, ==, dnodesize); ASSERT3U(bt->bt_offset, ==, offset); ASSERT3U(bt->bt_gen, <=, gen); ASSERT3U(bt->bt_txg, <=, txg); @@ -1452,6 +1494,52 @@ ztest_bt_bonus(dmu_buf_t *db) return (bt); } +/* + * Generate a token to fill up unused bonus buffer space. Try to make + * it unique to the object, generation, and offset to verify that data + * is not getting overwritten by data from other dnodes. + */ +#define ZTEST_BONUS_FILL_TOKEN(obj, ds, gen, offset) \ + (((ds) << 48) | ((gen) << 32) | ((obj) << 8) | (offset)) + +/* + * Fill up the unused bonus buffer region before the block tag with a + * verifiable pattern. Filling the whole bonus area with non-zero data + * helps ensure that all dnode traversal code properly skips the + * interior regions of large dnodes. + */ +void +ztest_fill_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj, + objset_t *os, uint64_t gen) +{ + uint64_t *bonusp; + + ASSERT(IS_P2ALIGNED((char *)end - (char *)db->db_data, 8)); + + for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) { + uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os), + gen, bonusp - (uint64_t *)db->db_data); + *bonusp = token; + } +} + +/* + * Verify that the unused area of a bonus buffer is filled with the + * expected tokens. + */ +void +ztest_verify_unused_bonus(dmu_buf_t *db, void *end, uint64_t obj, + objset_t *os, uint64_t gen) +{ + uint64_t *bonusp; + + for (bonusp = db->db_data; bonusp < (uint64_t *)end; bonusp++) { + uint64_t token = ZTEST_BONUS_FILL_TOKEN(obj, dmu_objset_id(os), + gen, bonusp - (uint64_t *)db->db_data); + VERIFY3U(*bonusp, ==, token); + } +} + /* * ZIL logging ops */ @@ -1460,7 +1548,7 @@ ztest_bt_bonus(dmu_buf_t *db) #define lrz_blocksize lr_uid #define lrz_ibshift lr_gid #define lrz_bonustype lr_rdev -#define lrz_bonuslen lr_crtime[1] +#define lrz_dnodesize lr_crtime[1] static void ztest_log_create(ztest_ds_t *zd, dmu_tx_t *tx, lr_create_t *lr) @@ -1575,6 +1663,7 @@ ztest_replay_create(ztest_ds_t *zd, lr_create_t *lr, boolean_t byteswap) dmu_tx_t *tx; uint64_t txg; int error = 0; + int bonuslen; if (byteswap) byteswap_uint64_array(lr, sizeof (*lr)); @@ -1597,26 +1686,27 @@ ztest_replay_create(ztest_ds_t *zd, lr_create_t *lr, boolean_t byteswap) return (ENOSPC); ASSERT(dmu_objset_zil(os)->zl_replay == !!lr->lr_foid); + bonuslen = DN_BONUS_SIZE(lr->lrz_dnodesize); if (lr->lrz_type == DMU_OT_ZAP_OTHER) { if (lr->lr_foid == 0) { - lr->lr_foid = zap_create(os, + lr->lr_foid = zap_create_dnsize(os, lr->lrz_type, lr->lrz_bonustype, - lr->lrz_bonuslen, tx); + bonuslen, lr->lrz_dnodesize, tx); } else { - error = zap_create_claim(os, lr->lr_foid, + error = zap_create_claim_dnsize(os, lr->lr_foid, lr->lrz_type, lr->lrz_bonustype, - lr->lrz_bonuslen, tx); + bonuslen, lr->lrz_dnodesize, tx); } } else { if (lr->lr_foid == 0) { - lr->lr_foid = dmu_object_alloc(os, + lr->lr_foid = dmu_object_alloc_dnsize(os, lr->lrz_type, 0, lr->lrz_bonustype, - lr->lrz_bonuslen, tx); + bonuslen, lr->lrz_dnodesize, tx); } else { - error = dmu_object_claim(os, lr->lr_foid, + error = dmu_object_claim_dnsize(os, lr->lr_foid, lr->lrz_type, 0, lr->lrz_bonustype, - lr->lrz_bonuslen, tx); + bonuslen, lr->lrz_dnodesize, tx); } } @@ -1636,7 +1726,9 @@ ztest_replay_create(ztest_ds_t *zd, lr_create_t *lr, boolean_t byteswap) VERIFY3U(0, ==, dmu_bonus_hold(os, lr->lr_foid, FTAG, &db)); bbt = ztest_bt_bonus(db); dmu_buf_will_dirty(db, tx); - ztest_bt_generate(bbt, os, lr->lr_foid, -1ULL, lr->lr_gen, txg, txg); + ztest_bt_generate(bbt, os, lr->lr_foid, lr->lrz_dnodesize, -1ULL, + lr->lr_gen, txg, txg); + ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, lr->lr_gen); dmu_buf_rele(db, FTAG); VERIFY3U(0, ==, zap_add(os, lr->lr_doid, name, sizeof (uint64_t), 1, @@ -1782,7 +1874,7 @@ ztest_replay_write(ztest_ds_t *zd, lr_write_t *lr, boolean_t byteswap) VERIFY(dmu_read(os, lr->lr_foid, offset, sizeof (rbt), &rbt, prefetch) == 0); if (rbt.bt_magic == BT_MAGIC) { - ztest_bt_verify(&rbt, os, lr->lr_foid, + ztest_bt_verify(&rbt, os, lr->lr_foid, 0, offset, gen, txg, crtxg); } } @@ -1794,7 +1886,7 @@ ztest_replay_write(ztest_ds_t *zd, lr_write_t *lr, boolean_t byteswap) * as it was when the write was generated. */ if (zd->zd_zilog->zl_replay) { - ztest_bt_verify(bt, os, lr->lr_foid, offset, + ztest_bt_verify(bt, os, lr->lr_foid, 0, offset, MAX(gen, bt->bt_gen), MAX(txg, lrtxg), bt->bt_crtxg); } @@ -1803,7 +1895,8 @@ ztest_replay_write(ztest_ds_t *zd, lr_write_t *lr, boolean_t byteswap) * Set the bt's gen/txg to the bonus buffer's gen/txg * so that all of the usual ASSERTs will work. */ - ztest_bt_generate(bt, os, lr->lr_foid, offset, gen, txg, crtxg); + ztest_bt_generate(bt, os, lr->lr_foid, 0, offset, gen, txg, + crtxg); } if (abuf == NULL) { @@ -1871,7 +1964,7 @@ ztest_replay_setattr(ztest_ds_t *zd, lr_setattr_t *lr, boolean_t byteswap) dmu_tx_t *tx; dmu_buf_t *db; ztest_block_tag_t *bbt; - uint64_t txg, lrtxg, crtxg; + uint64_t txg, lrtxg, crtxg, dnodesize; if (byteswap) byteswap_uint64_array(lr, sizeof (*lr)); @@ -1894,6 +1987,7 @@ ztest_replay_setattr(ztest_ds_t *zd, lr_setattr_t *lr, boolean_t byteswap) ASSERT3U(bbt->bt_magic, ==, BT_MAGIC); crtxg = bbt->bt_crtxg; lrtxg = lr->lr_common.lrc_txg; + dnodesize = bbt->bt_dnodesize; if (zd->zd_zilog->zl_replay) { ASSERT(lr->lr_size != 0); @@ -1912,7 +2006,7 @@ ztest_replay_setattr(ztest_ds_t *zd, lr_setattr_t *lr, boolean_t byteswap) /* * Verify that the current bonus buffer is not newer than our txg. */ - ztest_bt_verify(bbt, os, lr->lr_foid, -1ULL, lr->lr_mode, + ztest_bt_verify(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode, MAX(txg, lrtxg), crtxg); dmu_buf_will_dirty(db, tx); @@ -1922,8 +2016,9 @@ ztest_replay_setattr(ztest_ds_t *zd, lr_setattr_t *lr, boolean_t byteswap) VERIFY0(dmu_set_bonus(db, lr->lr_size, tx)); bbt = ztest_bt_bonus(db); - ztest_bt_generate(bbt, os, lr->lr_foid, -1ULL, lr->lr_mode, txg, crtxg); - + ztest_bt_generate(bbt, os, lr->lr_foid, dnodesize, -1ULL, lr->lr_mode, + txg, crtxg); + ztest_fill_unused_bonus(db, bbt, lr->lr_foid, os, bbt->bt_gen); dmu_buf_rele(db, FTAG); (void) ztest_log_setattr(zd, tx, lr); @@ -2168,7 +2263,7 @@ ztest_create(ztest_ds_t *zd, ztest_od_t *od, int count) lr->lrz_blocksize = od->od_crblocksize; lr->lrz_ibshift = ztest_random_ibshift(); lr->lrz_bonustype = DMU_OT_UINT64_OTHER; - lr->lrz_bonuslen = dmu_bonus_max(); + lr->lrz_dnodesize = od->od_crdnodesize; lr->lr_gen = od->od_crgen; lr->lr_crtime[0] = time(NULL); @@ -2348,7 +2443,8 @@ ztest_io(ztest_ds_t *zd, uint64_t object, uint64_t offset) switch (io_type) { case ZTEST_IO_WRITE_TAG: - ztest_bt_generate(&wbt, zd->zd_os, object, offset, 0, 0, 0); + ztest_bt_generate(&wbt, zd->zd_os, object, doi.doi_dnodesize, + offset, 0, 0, 0); (void) ztest_write(zd, object, offset, sizeof (wbt), &wbt); break; @@ -2411,13 +2507,15 @@ ztest_io(ztest_ds_t *zd, uint64_t object, uint64_t offset) */ static void ztest_od_init(ztest_od_t *od, uint64_t id, char *tag, uint64_t index, - dmu_object_type_t type, uint64_t blocksize, uint64_t gen) + dmu_object_type_t type, uint64_t blocksize, uint64_t dnodesize, + uint64_t gen) { od->od_dir = ZTEST_DIROBJ; od->od_object = 0; od->od_crtype = type; od->od_crblocksize = blocksize ? blocksize : ztest_random_blocksize(); + od->od_crdnodesize = dnodesize ? dnodesize : ztest_random_dnodesize(); od->od_crgen = gen; od->od_type = DMU_OT_NONE; @@ -3407,7 +3505,7 @@ ztest_objset_destroy_cb(const char *name, void *arg) static boolean_t ztest_snapshot_create(char *osname, uint64_t id) { - char snapname[MAXNAMELEN]; + char snapname[ZFS_MAX_DATASET_NAME_LEN]; int error; (void) snprintf(snapname, sizeof (snapname), "%llu", (u_longlong_t)id); @@ -3427,10 +3525,10 @@ ztest_snapshot_create(char *osname, uint64_t id) static boolean_t ztest_snapshot_destroy(char *osname, uint64_t id) { - char snapname[MAXNAMELEN]; + char snapname[ZFS_MAX_DATASET_NAME_LEN]; int error; - (void) snprintf(snapname, MAXNAMELEN, "%s@%llu", osname, + (void) snprintf(snapname, sizeof (snapname), "%s@%llu", osname, (u_longlong_t)id); error = dsl_destroy_snapshot(snapname, B_FALSE); @@ -3447,16 +3545,15 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id) int iters; int error; objset_t *os, *os2; - char *name; + char name[ZFS_MAX_DATASET_NAME_LEN]; zilog_t *zilog; int i; zdtmp = umem_alloc(sizeof (ztest_ds_t), UMEM_NOFAIL); - name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); (void) rw_rdlock(&ztest_name_lock); - (void) snprintf(name, MAXNAMELEN, "%s/temp_%llu", + (void) snprintf(name, sizeof (name), "%s/temp_%llu", ztest_opts.zo_pool, (u_longlong_t)id); /* @@ -3542,7 +3639,6 @@ ztest_dmu_objset_create_destroy(ztest_ds_t *zd, uint64_t id) out: (void) rw_unlock(&ztest_name_lock); - umem_free(name, MAXNAMELEN); umem_free(zdtmp, sizeof (ztest_ds_t)); } @@ -3571,22 +3667,22 @@ ztest_dsl_dataset_cleanup(char *osname, uint64_t id) char *snap3name; int error; - snap1name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); - clone1name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); - snap2name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); - clone2name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); - snap3name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); - - (void) snprintf(snap1name, MAXNAMELEN, "%s@s1_%llu", - osname, (u_longlong_t)id); - (void) snprintf(clone1name, MAXNAMELEN, "%s/c1_%llu", - osname, (u_longlong_t)id); - (void) snprintf(snap2name, MAXNAMELEN, "%s@s2_%llu", - clone1name, (u_longlong_t)id); - (void) snprintf(clone2name, MAXNAMELEN, "%s/c2_%llu", - osname, (u_longlong_t)id); - (void) snprintf(snap3name, MAXNAMELEN, "%s@s3_%llu", - clone1name, (u_longlong_t)id); + snap1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); + clone1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); + snap2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); + clone2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); + snap3name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); + + (void) snprintf(snap1name, ZFS_MAX_DATASET_NAME_LEN, + "%s@s1_%llu", osname, (u_longlong_t)id); + (void) snprintf(clone1name, ZFS_MAX_DATASET_NAME_LEN, + "%s/c1_%llu", osname, (u_longlong_t)id); + (void) snprintf(snap2name, ZFS_MAX_DATASET_NAME_LEN, + "%s@s2_%llu", clone1name, (u_longlong_t)id); + (void) snprintf(clone2name, ZFS_MAX_DATASET_NAME_LEN, + "%s/c2_%llu", osname, (u_longlong_t)id); + (void) snprintf(snap3name, ZFS_MAX_DATASET_NAME_LEN, + "%s@s3_%llu", clone1name, (u_longlong_t)id); error = dsl_destroy_head(clone2name); if (error && error != ENOENT) @@ -3604,11 +3700,11 @@ ztest_dsl_dataset_cleanup(char *osname, uint64_t id) if (error && error != ENOENT) fatal(0, "dsl_destroy_snapshot(%s) = %d", snap1name, error); - umem_free(snap1name, MAXNAMELEN); - umem_free(clone1name, MAXNAMELEN); - umem_free(snap2name, MAXNAMELEN); - umem_free(clone2name, MAXNAMELEN); - umem_free(snap3name, MAXNAMELEN); + umem_free(snap1name, ZFS_MAX_DATASET_NAME_LEN); + umem_free(clone1name, ZFS_MAX_DATASET_NAME_LEN); + umem_free(snap2name, ZFS_MAX_DATASET_NAME_LEN); + umem_free(clone2name, ZFS_MAX_DATASET_NAME_LEN); + umem_free(snap3name, ZFS_MAX_DATASET_NAME_LEN); } /* @@ -3626,26 +3722,26 @@ ztest_dsl_dataset_promote_busy(ztest_ds_t *zd, uint64_t id) char *osname = zd->zd_name; int error; - snap1name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); - clone1name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); - snap2name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); - clone2name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); - snap3name = umem_alloc(MAXNAMELEN, UMEM_NOFAIL); + snap1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); + clone1name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); + snap2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); + clone2name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); + snap3name = umem_alloc(ZFS_MAX_DATASET_NAME_LEN, UMEM_NOFAIL); (void) rw_rdlock(&ztest_name_lock); ztest_dsl_dataset_cleanup(osname, id); - (void) snprintf(snap1name, MAXNAMELEN, "%s@s1_%llu", - osname, (u_longlong_t)id); - (void) snprintf(clone1name, MAXNAMELEN, "%s/c1_%llu", - osname, (u_longlong_t)id); - (void) snprintf(snap2name, MAXNAMELEN, "%s@s2_%llu", - clone1name, (u_longlong_t)id); - (void) snprintf(clone2name, MAXNAMELEN, "%s/c2_%llu", - osname, (u_longlong_t)id); - (void) snprintf(snap3name, MAXNAMELEN, "%s@s3_%llu", - clone1name, (u_longlong_t)id); + (void) snprintf(snap1name, ZFS_MAX_DATASET_NAME_LEN, + "%s@s1_%llu", osname, (u_longlong_t)id); + (void) snprintf(clone1name, ZFS_MAX_DATASET_NAME_LEN, + "%s/c1_%llu", osname, (u_longlong_t)id); + (void) snprintf(snap2name, ZFS_MAX_DATASET_NAME_LEN, + "%s@s2_%llu", clone1name, (u_longlong_t)id); + (void) snprintf(clone2name, ZFS_MAX_DATASET_NAME_LEN, + "%s/c2_%llu", osname, (u_longlong_t)id); + (void) snprintf(snap3name, ZFS_MAX_DATASET_NAME_LEN, + "%s@s3_%llu", clone1name, (u_longlong_t)id); error = dmu_objset_snapshot_one(osname, strchr(snap1name, '@') + 1); if (error && error != EEXIST) { @@ -3711,11 +3807,11 @@ ztest_dsl_dataset_promote_busy(ztest_ds_t *zd, uint64_t id) (void) rw_unlock(&ztest_name_lock); - umem_free(snap1name, MAXNAMELEN); - umem_free(clone1name, MAXNAMELEN); - umem_free(snap2name, MAXNAMELEN); - umem_free(clone2name, MAXNAMELEN); - umem_free(snap3name, MAXNAMELEN); + umem_free(snap1name, ZFS_MAX_DATASET_NAME_LEN); + umem_free(clone1name, ZFS_MAX_DATASET_NAME_LEN); + umem_free(snap2name, ZFS_MAX_DATASET_NAME_LEN); + umem_free(clone2name, ZFS_MAX_DATASET_NAME_LEN); + umem_free(snap3name, ZFS_MAX_DATASET_NAME_LEN); } #undef OD_ARRAY_SIZE @@ -3737,7 +3833,8 @@ ztest_dmu_object_alloc_free(ztest_ds_t *zd, uint64_t id) batchsize = OD_ARRAY_SIZE; for (b = 0; b < batchsize; b++) - ztest_od_init(od + b, id, FTAG, b, DMU_OT_UINT64_OTHER, 0, 0); + ztest_od_init(od + b, id, FTAG, b, DMU_OT_UINT64_OTHER, + 0, 0, 0); /* * Destroy the previous batch of objects, create a new batch, @@ -3806,8 +3903,9 @@ ztest_dmu_read_write(ztest_ds_t *zd, uint64_t id) /* * Read the directory info. If it's the first time, set things up. */ - ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, chunksize); - ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, chunksize); + ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, chunksize); + ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0, + chunksize); if (ztest_object_init(zd, od, size, B_FALSE) != 0) { umem_free(od, size); @@ -4087,8 +4185,9 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id) /* * Read the directory info. If it's the first time, set things up. */ - ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0); - ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, chunksize); + ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0); + ztest_od_init(od + 1, id, FTAG, 1, DMU_OT_UINT64_OTHER, 0, 0, + chunksize); if (ztest_object_init(zd, od, size, B_FALSE) != 0) { @@ -4296,7 +4395,7 @@ ztest_dmu_write_parallel(ztest_ds_t *zd, uint64_t id) * to verify that parallel writes to an object -- even to the * same blocks within the object -- doesn't cause any trouble. */ - ztest_od_init(od, ID_PARALLEL, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0); + ztest_od_init(od, ID_PARALLEL, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0); if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) return; @@ -4319,7 +4418,7 @@ ztest_dmu_prealloc(ztest_ds_t *zd, uint64_t id) od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL); - ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0); + ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0); if (ztest_object_init(zd, od, sizeof (ztest_od_t), !ztest_random(2)) != 0) { @@ -4372,7 +4471,7 @@ ztest_zap(ztest_ds_t *zd, uint64_t id) char *hc[2] = { "s.acl.h", ".s.open.h.hyLZlg" }; od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL); - ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0); + ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0); if (ztest_object_init(zd, od, sizeof (ztest_od_t), !ztest_random(2)) != 0) @@ -4509,7 +4608,7 @@ ztest_fzap(ztest_ds_t *zd, uint64_t id) int i; od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL); - ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0); + ztest_od_init(od, id, FTAG, 0, DMU_OT_ZAP_OTHER, 0, 0, 0); if (ztest_object_init(zd, od, sizeof (ztest_od_t), !ztest_random(2)) != 0) @@ -4522,7 +4621,7 @@ ztest_fzap(ztest_ds_t *zd, uint64_t id) * 2050 entries we should see ptrtbl growth and leaf-block split. */ for (i = 0; i < 2050; i++) { - char name[MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; uint64_t value = i; dmu_tx_t *tx; int error; @@ -4558,7 +4657,7 @@ ztest_zap_parallel(ztest_ds_t *zd, uint64_t id) void *data; od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL); - ztest_od_init(od, ID_PARALLEL, FTAG, micro, DMU_OT_ZAP_OTHER, 0, 0); + ztest_od_init(od, ID_PARALLEL, FTAG, micro, DMU_OT_ZAP_OTHER, 0, 0, 0); if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) { umem_free(od, sizeof (ztest_od_t)); @@ -4604,8 +4703,10 @@ ztest_zap_parallel(ztest_ds_t *zd, uint64_t id) tx = dmu_tx_create(os); dmu_tx_hold_zap(tx, object, B_TRUE, NULL); txg = ztest_tx_assign(tx, TXG_MIGHTWAIT, FTAG); - if (txg == 0) + if (txg == 0) { + umem_free(od, sizeof (ztest_od_t)); return; + } bcopy(name, string_value, namelen); } else { tx = NULL; @@ -4747,7 +4848,7 @@ ztest_dmu_commit_callbacks(ztest_ds_t *zd, uint64_t id) int i, error = 0; od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL); - ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0); + ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0); if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) { umem_free(od, sizeof (ztest_od_t)); @@ -4868,6 +4969,41 @@ ztest_dmu_commit_callbacks(ztest_ds_t *zd, uint64_t id) umem_free(od, sizeof (ztest_od_t)); } +/* + * Visit each object in the dataset. Verify that its properties + * are consistent what was stored in the block tag when it was created, + * and that its unused bonus buffer space has not been overwritten. + */ +void +ztest_verify_dnode_bt(ztest_ds_t *zd, uint64_t id) +{ + objset_t *os = zd->zd_os; + uint64_t obj; + int err = 0; + + for (obj = 0; err == 0; err = dmu_object_next(os, &obj, FALSE, 0)) { + ztest_block_tag_t *bt = NULL; + dmu_object_info_t doi; + dmu_buf_t *db; + + if (dmu_bonus_hold(os, obj, FTAG, &db) != 0) + continue; + + dmu_object_info_from_db(db, &doi); + if (doi.doi_bonus_size >= sizeof (*bt)) + bt = ztest_bt_bonus(db); + + if (bt && bt->bt_magic == BT_MAGIC) { + ztest_bt_verify(bt, os, obj, doi.doi_dnodesize, + bt->bt_offset, bt->bt_gen, bt->bt_txg, + bt->bt_crtxg); + ztest_verify_unused_bonus(db, bt, obj, os, bt->bt_gen); + } + + dmu_buf_rele(db, FTAG); + } +} + /* ARGSUSED */ void ztest_dsl_prop_get_set(ztest_ds_t *zd, uint64_t id) @@ -4942,7 +5078,7 @@ ztest_dmu_snapshot_hold(ztest_ds_t *zd, uint64_t id) char fullname[100]; char clonename[100]; char tag[100]; - char osname[MAXNAMELEN]; + char osname[ZFS_MAX_DATASET_NAME_LEN]; nvlist_t *holds; (void) rw_rdlock(&ztest_name_lock); @@ -5314,7 +5450,7 @@ ztest_ddt_repair(ztest_ds_t *zd, uint64_t id) blocksize = MIN(blocksize, 2048); /* because we write so many */ od = umem_alloc(sizeof (ztest_od_t), UMEM_NOFAIL); - ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0); + ztest_od_init(od, id, FTAG, 0, DMU_OT_UINT64_OTHER, blocksize, 0, 0); if (ztest_object_init(zd, od, sizeof (ztest_od_t), B_FALSE) != 0) { umem_free(od, sizeof (ztest_od_t)); @@ -5496,6 +5632,47 @@ ztest_spa_rename(ztest_ds_t *zd, uint64_t id) (void) rw_unlock(&ztest_name_lock); } +void +ztest_fletcher(ztest_ds_t *zd, uint64_t id) +{ + hrtime_t end = gethrtime() + NANOSEC; + + while (gethrtime() <= end) { + int run_count = 100; + void *buf; + uint32_t size; + int *ptr; + int i; + zio_cksum_t zc_ref; + zio_cksum_t zc_ref_byteswap; + + size = ztest_random_blocksize(); + buf = umem_alloc(size, UMEM_NOFAIL); + + for (i = 0, ptr = buf; i < size / sizeof (*ptr); i++, ptr++) + *ptr = ztest_random(UINT_MAX); + + VERIFY0(fletcher_4_impl_set("scalar")); + fletcher_4_native(buf, size, &zc_ref); + fletcher_4_byteswap(buf, size, &zc_ref_byteswap); + + VERIFY0(fletcher_4_impl_set("cycle")); + while (run_count-- > 0) { + zio_cksum_t zc; + zio_cksum_t zc_byteswap; + + fletcher_4_byteswap(buf, size, &zc_byteswap); + fletcher_4_native(buf, size, &zc); + + VERIFY0(bcmp(&zc, &zc_ref, sizeof (zc))); + VERIFY0(bcmp(&zc_byteswap, &zc_ref_byteswap, + sizeof (zc_byteswap))); + } + + umem_free(buf, size); + } +} + static int ztest_check_path(char *path) { @@ -5788,13 +5965,13 @@ ztest_thread(void *arg) static void ztest_dataset_name(char *dsname, char *pool, int d) { - (void) snprintf(dsname, MAXNAMELEN, "%s/ds_%d", pool, d); + (void) snprintf(dsname, ZFS_MAX_DATASET_NAME_LEN, "%s/ds_%d", pool, d); } static void ztest_dataset_destroy(int d) { - char name[MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; int t; ztest_dataset_name(name, ztest_opts.zo_pool, d); @@ -5843,7 +6020,7 @@ ztest_dataset_open(int d) uint64_t committed_seq = ZTEST_GET_SHARED_DS(d)->zd_seq; objset_t *os; zilog_t *zilog; - char name[MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; int error; ztest_dataset_name(name, ztest_opts.zo_pool, d); @@ -6080,8 +6257,8 @@ ztest_run(ztest_shared_t *zs) * different name. */ if (ztest_random(2) == 0) { - char name[MAXNAMELEN]; - (void) snprintf(name, MAXNAMELEN, "%s_import", + char name[ZFS_MAX_DATASET_NAME_LEN]; + (void) snprintf(name, sizeof (name), "%s_import", ztest_opts.zo_pool); ztest_spa_import_export(ztest_opts.zo_pool, name); ztest_spa_import_export(name, ztest_opts.zo_pool); @@ -6150,7 +6327,7 @@ ztest_freeze(void) numloops++ < ztest_opts.zo_maxloops && metaslab_class_get_alloc(spa_normal_class(spa)) < capacity) { ztest_od_t od; - ztest_od_init(&od, 0, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0); + ztest_od_init(&od, 0, FTAG, 0, DMU_OT_UINT64_OTHER, 0, 0, 0); VERIFY0(ztest_object_init(zd, &od, sizeof (od), B_FALSE)); ztest_io(zd, od.od_object, ztest_random(ZTEST_RANGE_LOCKS) << SPA_MAXBLOCKSHIFT); @@ -6658,7 +6835,7 @@ main(int argc, char **argv) if (spa_open(ztest_opts.zo_pool, &spa, FTAG) == 0) { spa_close(spa, FTAG); } else { - char tmpname[MAXNAMELEN]; + char tmpname[ZFS_MAX_DATASET_NAME_LEN]; kernel_fini(); kernel_init(FREAD | FWRITE); (void) snprintf(tmpname, sizeof (tmpname), "%s_tmp", diff --git a/cmd/zvol_id/zvol_id_main.c b/cmd/zvol_id/zvol_id_main.c index 8f0504ae9992..6bd5113f1ea7 100644 --- a/cmd/zvol_id/zvol_id_main.c +++ b/cmd/zvol_id/zvol_id_main.c @@ -39,14 +39,14 @@ static int ioctl_get_msg(char *var, int fd) { int error = 0; - char msg[ZFS_MAXNAMELEN]; + char msg[ZFS_MAX_DATASET_NAME_LEN]; error = ioctl(fd, BLKZNAME, msg); if (error < 0) { return (error); } - snprintf(var, ZFS_MAXNAMELEN, "%s", msg); + snprintf(var, ZFS_MAX_DATASET_NAME_LEN, "%s", msg); return (error); } @@ -54,7 +54,8 @@ int main(int argc, char **argv) { int fd, error = 0; - char zvol_name[ZFS_MAXNAMELEN], zvol_name_part[ZFS_MAXNAMELEN]; + char zvol_name[ZFS_MAX_DATASET_NAME_LEN]; + char zvol_name_part[ZFS_MAX_DATASET_NAME_LEN]; char *dev_name; struct stat64 statbuf; int dev_minor, dev_part; @@ -87,10 +88,11 @@ main(int argc, char **argv) return (errno); } if (dev_part > 0) - snprintf(zvol_name_part, ZFS_MAXNAMELEN, "%s-part%d", zvol_name, - dev_part); + snprintf(zvol_name_part, ZFS_MAX_DATASET_NAME_LEN, + "%s-part%d", zvol_name, dev_part); else - snprintf(zvol_name_part, ZFS_MAXNAMELEN, "%s", zvol_name); + snprintf(zvol_name_part, ZFS_MAX_DATASET_NAME_LEN, + "%s", zvol_name); for (i = 0; i < strlen(zvol_name_part); i++) { if (isblank(zvol_name_part[i])) diff --git a/config/always-arch.m4 b/config/always-arch.m4 new file mode 100644 index 000000000000..c3e6b4a9789a --- /dev/null +++ b/config/always-arch.m4 @@ -0,0 +1,22 @@ +dnl # +dnl # Set the target arch for libspl atomic implementation and the icp +dnl # +AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_ARCH], [ + AC_MSG_CHECKING(for target asm dir) + TARGET_ARCH=`echo ${target_cpu} | sed -e s/i.86/i386/` + + case $TARGET_ARCH in + i386|x86_64) + TARGET_ASM_DIR=asm-${TARGET_ARCH} + ;; + *) + TARGET_ASM_DIR=asm-generic + ;; + esac + + AC_SUBST([TARGET_ASM_DIR]) + AM_CONDITIONAL([TARGET_ASM_X86_64], test $TARGET_ASM_DIR = asm-x86_64) + AM_CONDITIONAL([TARGET_ASM_I386], test $TARGET_ASM_DIR = asm-i386) + AM_CONDITIONAL([TARGET_ASM_GENERIC], test $TARGET_ASM_DIR = asm-generic) + AC_MSG_RESULT([$TARGET_ASM_DIR]) +]) diff --git a/config/kernel-blk-queue-unplug.m4 b/config/kernel-blk-queue-unplug.m4 new file mode 100644 index 000000000000..45cc2322a5e4 --- /dev/null +++ b/config/kernel-blk-queue-unplug.m4 @@ -0,0 +1,23 @@ +dnl # +dnl # 2.6.32-2.6.35 API - The BIO_RW_UNPLUG enum can be used as a hint +dnl # to unplug the queue. +dnl # +AC_DEFUN([ZFS_AC_KERNEL_BLK_QUEUE_HAVE_BIO_RW_UNPLUG], [ + AC_MSG_CHECKING([whether the BIO_RW_UNPLUG enum is available]) + tmp_flags="$EXTRA_KCFLAGS" + EXTRA_KCFLAGS="${NO_UNUSED_BUT_SET_VARIABLE}" + ZFS_LINUX_TRY_COMPILE([ + #include + ],[ + extern enum bio_rw_flags rw; + + rw = BIO_RW_UNPLUG; + ],[ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_BLK_QUEUE_HAVE_BIO_RW_UNPLUG, 1, + [BIO_RW_UNPLUG is available]) + ],[ + AC_MSG_RESULT(no) + ]) + EXTRA_KCFLAGS="$tmp_flags" +]) diff --git a/config/kernel-kobj-name-len.m4 b/config/kernel-kobj-name-len.m4 deleted file mode 100644 index 37999fabb81d..000000000000 --- a/config/kernel-kobj-name-len.m4 +++ /dev/null @@ -1,21 +0,0 @@ -dnl # -dnl # 2.6.27 API change, -dnl # kobject KOBJ_NAME_LEN static limit removed. All users of this -dnl # constant were removed prior to 2.6.27, but to be on the safe -dnl # side this check ensures the constant is undefined. -dnl # -AC_DEFUN([ZFS_AC_KERNEL_KOBJ_NAME_LEN], [ - AC_MSG_CHECKING([whether kernel defines KOBJ_NAME_LEN]) - ZFS_LINUX_TRY_COMPILE([ - #include - ],[ - int val __attribute__ ((unused)); - val = KOBJ_NAME_LEN; - ],[ - AC_MSG_RESULT([yes]) - AC_DEFINE(HAVE_KOBJ_NAME_LEN, 1, - [kernel defines KOBJ_NAME_LEN]) - ],[ - AC_MSG_RESULT([no]) - ]) -]) diff --git a/config/kernel-kuid-helpers.m4 b/config/kernel-kuid-helpers.m4 new file mode 100644 index 000000000000..60713b9d3132 --- /dev/null +++ b/config/kernel-kuid-helpers.m4 @@ -0,0 +1,22 @@ +dnl # +dnl # 3.5 API change, +dnl # Since usernamespaces were introduced in kernel version 3.5, it +dnl # became necessary to go through one more level of indirection +dnl # when dealing with uid/gid - namely the kuid type. +dnl # +dnl # +AC_DEFUN([ZFS_AC_KERNEL_KUID_HELPERS], [ + AC_MSG_CHECKING([whether i_(uid|gid)_(read|write) exist]) + ZFS_LINUX_TRY_COMPILE([ + #include + ],[ + struct inode *ip = NULL; + (void) i_uid_read(ip); + ],[ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_KUID_HELPERS, 1, + [i_(uid|gid)_(read|write) exist]) + ],[ + AC_MSG_RESULT(no) + ]) +]) diff --git a/config/kernel-submit_bio.m4 b/config/kernel-submit_bio.m4 new file mode 100644 index 000000000000..da5f85ca72cb --- /dev/null +++ b/config/kernel-submit_bio.m4 @@ -0,0 +1,20 @@ +dnl # +dnl # 4.8 API change +dnl # The rw argument has been removed from submit_bio/submit_bio_wait. +dnl # Callers are now expected to set bio->bi_rw instead of passing it in. +dnl # +AC_DEFUN([ZFS_AC_KERNEL_SUBMIT_BIO], [ + AC_MSG_CHECKING([whether submit_bio() wants 1 arg]) + ZFS_LINUX_TRY_COMPILE([ + #include + ],[ + blk_qc_t blk_qc; + struct bio *bio = NULL; + blk_qc = submit_bio(bio); + ],[ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_1ARG_SUBMIT_BIO, 1, [submit_bio() wants 1 arg]) + ],[ + AC_MSG_RESULT(no) + ]) +]) diff --git a/config/kernel-xattr-handler.m4 b/config/kernel-xattr-handler.m4 index 638557e32e40..ce18eaf9caf6 100644 --- a/config/kernel-xattr-handler.m4 +++ b/config/kernel-xattr-handler.m4 @@ -81,7 +81,7 @@ AC_DEFUN([ZFS_AC_KERNEL_XATTR_HANDLER_GET], [ ],[ AC_MSG_RESULT(yes) AC_DEFINE(HAVE_XATTR_GET_DENTRY_INODE, 1, - [xattr_handler->get() wants xattr_handler]) + [xattr_handler->get() wants both dentry and inode]) ],[ dnl # dnl # 4.4 API change, @@ -89,6 +89,7 @@ AC_DEFUN([ZFS_AC_KERNEL_XATTR_HANDLER_GET], [ dnl # attr_handler, and handler_flags argument was removed and dnl # should be accessed by handler->flags. dnl # + AC_MSG_RESULT(no) AC_MSG_CHECKING([whether xattr_handler->get() wants xattr_handler]) ZFS_LINUX_TRY_COMPILE([ #include @@ -163,18 +164,18 @@ dnl # Supported xattr handler set() interfaces checked newest to oldest. dnl # AC_DEFUN([ZFS_AC_KERNEL_XATTR_HANDLER_SET], [ dnl # - dnl # 4.4 API change, - dnl # The xattr_handler->set() callback was changed to take a - dnl # xattr_handler, and handler_flags argument was removed and - dnl # should be accessed by handler->flags. + dnl # 4.7 API change, + dnl # The xattr_handler->set() callback was changed to take both + dnl # dentry and inode. dnl # - AC_MSG_CHECKING([whether xattr_handler->set() wants xattr_handler]) + AC_MSG_CHECKING([whether xattr_handler->set() wants both dentry and inode]) ZFS_LINUX_TRY_COMPILE([ #include int set(const struct xattr_handler *handler, - struct dentry *dentry, const char *name, - const void *buffer, size_t size, int flags) + struct dentry *dentry, struct inode *inode, + const char *name, const void *buffer, + size_t size, int flags) { return 0; } static const struct xattr_handler xops __attribute__ ((unused)) = { @@ -183,23 +184,24 @@ AC_DEFUN([ZFS_AC_KERNEL_XATTR_HANDLER_SET], [ ],[ ],[ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_XATTR_SET_HANDLER, 1, - [xattr_handler->set() wants xattr_handler]) + AC_DEFINE(HAVE_XATTR_SET_DENTRY_INODE, 1, + [xattr_handler->set() wants both dentry and inode]) ],[ dnl # - dnl # 2.6.33 API change, + dnl # 4.4 API change, dnl # The xattr_handler->set() callback was changed to take a - dnl # dentry instead of an inode, and a handler_flags - dnl # argument was added. + dnl # xattr_handler, and handler_flags argument was removed and + dnl # should be accessed by handler->flags. dnl # AC_MSG_RESULT(no) - AC_MSG_CHECKING([whether xattr_handler->set() wants dentry]) + AC_MSG_CHECKING([whether xattr_handler->set() wants xattr_handler]) ZFS_LINUX_TRY_COMPILE([ #include - int set(struct dentry *dentry, const char *name, - const void *buffer, size_t size, int flags, - int handler_flags) { return 0; } + int set(const struct xattr_handler *handler, + struct dentry *dentry, const char *name, + const void *buffer, size_t size, int flags) + { return 0; } static const struct xattr_handler xops __attribute__ ((unused)) = { .set = set, @@ -207,21 +209,23 @@ AC_DEFUN([ZFS_AC_KERNEL_XATTR_HANDLER_SET], [ ],[ ],[ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_XATTR_SET_DENTRY, 1, - [xattr_handler->set() wants dentry]) + AC_DEFINE(HAVE_XATTR_SET_HANDLER, 1, + [xattr_handler->set() wants xattr_handler]) ],[ dnl # - dnl # 2.6.32 API + dnl # 2.6.33 API change, + dnl # The xattr_handler->set() callback was changed to take a + dnl # dentry instead of an inode, and a handler_flags + dnl # argument was added. dnl # AC_MSG_RESULT(no) - AC_MSG_CHECKING( - [whether xattr_handler->set() wants inode]) + AC_MSG_CHECKING([whether xattr_handler->set() wants dentry]) ZFS_LINUX_TRY_COMPILE([ #include - int set(struct inode *ip, const char *name, - const void *buffer, size_t size, int flags) - { return 0; } + int set(struct dentry *dentry, const char *name, + const void *buffer, size_t size, int flags, + int handler_flags) { return 0; } static const struct xattr_handler xops __attribute__ ((unused)) = { .set = set, @@ -229,10 +233,33 @@ AC_DEFUN([ZFS_AC_KERNEL_XATTR_HANDLER_SET], [ ],[ ],[ AC_MSG_RESULT(yes) - AC_DEFINE(HAVE_XATTR_SET_INODE, 1, - [xattr_handler->set() wants inode]) + AC_DEFINE(HAVE_XATTR_SET_DENTRY, 1, + [xattr_handler->set() wants dentry]) ],[ - AC_MSG_ERROR([no; please file a bug report]) + dnl # + dnl # 2.6.32 API + dnl # + AC_MSG_RESULT(no) + AC_MSG_CHECKING( + [whether xattr_handler->set() wants inode]) + ZFS_LINUX_TRY_COMPILE([ + #include + + int set(struct inode *ip, const char *name, + const void *buffer, size_t size, int flags) + { return 0; } + static const struct xattr_handler + xops __attribute__ ((unused)) = { + .set = set, + }; + ],[ + ],[ + AC_MSG_RESULT(yes) + AC_DEFINE(HAVE_XATTR_SET_INODE, 1, + [xattr_handler->set() wants inode]) + ],[ + AC_MSG_ERROR([no; please file a bug report]) + ]) ]) ]) ]) diff --git a/config/kernel.m4 b/config/kernel.m4 index 800d782f1eee..7edafee0fc20 100644 --- a/config/kernel.m4 +++ b/config/kernel.m4 @@ -8,10 +8,10 @@ AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [ ZFS_AC_KERNEL_CONFIG ZFS_AC_KERNEL_DECLARE_EVENT_CLASS ZFS_AC_KERNEL_CURRENT_BIO_TAIL + ZFS_AC_KERNEL_SUBMIT_BIO ZFS_AC_KERNEL_BDEV_BLOCK_DEVICE_OPERATIONS ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS_RELEASE_VOID ZFS_AC_KERNEL_TYPE_FMODE_T - ZFS_AC_KERNEL_KOBJ_NAME_LEN ZFS_AC_KERNEL_3ARG_BLKDEV_GET ZFS_AC_KERNEL_BLKDEV_GET_BY_PATH ZFS_AC_KERNEL_OPEN_BDEV_EXCLUSIVE @@ -28,6 +28,7 @@ AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [ ZFS_AC_KERNEL_BLK_QUEUE_FLUSH ZFS_AC_KERNEL_BLK_QUEUE_MAX_HW_SECTORS ZFS_AC_KERNEL_BLK_QUEUE_MAX_SEGMENTS + ZFS_AC_KERNEL_BLK_QUEUE_HAVE_BIO_RW_UNPLUG ZFS_AC_KERNEL_GET_DISK_RO ZFS_AC_KERNEL_GET_GENDISK ZFS_AC_KERNEL_DISCARD_GRANULARITY @@ -93,6 +94,7 @@ AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [ ZFS_AC_KERNEL_MAKE_REQUEST_FN ZFS_AC_KERNEL_GENERIC_IO_ACCT ZFS_AC_KERNEL_FPU + ZFS_AC_KERNEL_KUID_HELPERS AS_IF([test "$LINUX_OBJ" != "$LINUX"], [ KERNELMAKE_PARAMS="$KERNELMAKE_PARAMS O=$LINUX_OBJ" diff --git a/config/user-arch.m4 b/config/user-arch.m4 deleted file mode 100644 index fcc566fc5db5..000000000000 --- a/config/user-arch.m4 +++ /dev/null @@ -1,19 +0,0 @@ -dnl # -dnl # Set the target arch for libspl atomic implementation -dnl # -AC_DEFUN([ZFS_AC_CONFIG_USER_ARCH], [ - AC_MSG_CHECKING(for target asm dir) - TARGET_ARCH=`echo ${target_cpu} | sed -e s/i.86/i386/` - - case $TARGET_ARCH in - i386|x86_64) - TARGET_ASM_DIR=asm-${TARGET_ARCH} - ;; - *) - TARGET_ASM_DIR=asm-generic - ;; - esac - - AC_SUBST([TARGET_ASM_DIR]) - AC_MSG_RESULT([$TARGET_ASM_DIR]) -]) diff --git a/config/user-commands.m4 b/config/user-commands.m4 index 655b992418cc..bda2b8652b53 100644 --- a/config/user-commands.m4 +++ b/config/user-commands.m4 @@ -67,6 +67,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_COMMON], [ AC_PATH_TOOL(SHUF, shuf, "") AC_PATH_TOOL(SLEEP, sleep, "") AC_PATH_TOOL(SORT, sort, "") + AC_PATH_TOOL(STAT, stat, "") AC_PATH_TOOL(STRINGS, strings, "") AC_PATH_TOOL(SU, su, "") AC_PATH_TOOL(SUM, sum, "") @@ -75,6 +76,7 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_COMMON], [ AC_PATH_TOOL(TAR, tar, "") AC_PATH_TOOL(TOUCH, touch, "") AC_PATH_TOOL(TR, tr, "") + AC_PATH_TOOL(TRUNCATE, truncate, "") AC_PATH_TOOL(TRUE, true, "") AC_PATH_TOOL(UMASK, umask, "") AC_PATH_TOOL(UMOUNT, umount, "") @@ -103,7 +105,6 @@ AC_DEFUN([ZFS_AC_CONFIG_USER_COMMANDS_LINUX], [ AC_PATH_TOOL(SHARE, exportfs, "") AC_PATH_TOOL(SWAP, swapon, "") AC_PATH_TOOL(SWAPADD, swapon, "") - AC_PATH_TOOL(TRUNCATE, truncate, "") AC_PATH_TOOL(UDEVADM, udevadm, "") AC_PATH_TOOL(UFSDUMP, dump, "") AC_PATH_TOOL(UFSRESTORE, restore, "") diff --git a/config/user.m4 b/config/user.m4 index 4081c98a9695..f70ab635fa5b 100644 --- a/config/user.m4 +++ b/config/user.m4 @@ -7,7 +7,6 @@ AC_DEFUN([ZFS_AC_CONFIG_USER], [ ZFS_AC_CONFIG_USER_SYSTEMD ZFS_AC_CONFIG_USER_SYSVINIT ZFS_AC_CONFIG_USER_DRACUT - ZFS_AC_CONFIG_USER_ARCH ZFS_AC_CONFIG_USER_ZLIB ZFS_AC_CONFIG_USER_LIBUUID ZFS_AC_CONFIG_USER_LIBTIRPC diff --git a/config/zfs-build.m4 b/config/zfs-build.m4 index a584b081360e..8b969da36ff5 100644 --- a/config/zfs-build.m4 +++ b/config/zfs-build.m4 @@ -64,12 +64,10 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [ ZFS_AC_CONFIG_ALWAYS_NO_UNUSED_BUT_SET_VARIABLE ZFS_AC_CONFIG_ALWAYS_NO_BOOL_COMPARE ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD + ZFS_AC_CONFIG_ALWAYS_ARCH ]) AC_DEFUN([ZFS_AC_CONFIG], [ - TARGET_ASM_DIR=asm-generic - AC_SUBST(TARGET_ASM_DIR) - ZFS_CONFIG=all AC_ARG_WITH([config], AS_HELP_STRING([--with-config=CONFIG], diff --git a/configure.ac b/configure.ac index ce08089724a6..9a8d6d24b836 100644 --- a/configure.ac +++ b/configure.ac @@ -39,7 +39,7 @@ AC_CONFIG_MACRO_DIR([config]) AC_CANONICAL_SYSTEM AM_MAINTAINER_MODE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) -AM_INIT_AUTOMAKE +AM_INIT_AUTOMAKE([subdir-objects]) AC_CONFIG_HEADERS([zfs_config.h], [ (mv zfs_config.h zfs_config.h.tmp && awk -f ${ac_srcdir}/config/config.awk zfs_config.h.tmp >zfs_config.h && @@ -85,6 +85,7 @@ AC_CONFIG_FILES([ lib/libspl/include/util/Makefile lib/libavl/Makefile lib/libefi/Makefile + lib/libicp/Makefile lib/libnvpair/Makefile lib/libunicode/Makefile lib/libuutil/Makefile @@ -111,6 +112,7 @@ AC_CONFIG_FILES([ cmd/dbufstat/Makefile cmd/arc_summary/Makefile cmd/zed/Makefile + cmd/raidz_test/Makefile contrib/Makefile contrib/bash_completion.d/Makefile contrib/dracut/Makefile @@ -124,12 +126,14 @@ AC_CONFIG_FILES([ module/zcommon/Makefile module/zfs/Makefile module/zpios/Makefile + module/icp/Makefile include/Makefile include/linux/Makefile include/sys/Makefile include/sys/fs/Makefile include/sys/fm/Makefile include/sys/fm/fs/Makefile + include/sys/crypto/Makefile scripts/Makefile scripts/zpios-profile/Makefile scripts/zpios-test/Makefile @@ -229,6 +233,7 @@ AC_CONFIG_FILES([ tests/zfs-tests/tests/functional/devices/Makefile tests/zfs-tests/tests/functional/exec/Makefile tests/zfs-tests/tests/functional/features/async_destroy/Makefile + tests/zfs-tests/tests/functional/features/large_dnode/Makefile tests/zfs-tests/tests/functional/features/Makefile tests/zfs-tests/tests/functional/grow_pool/Makefile tests/zfs-tests/tests/functional/grow_replicas/Makefile @@ -250,6 +255,7 @@ AC_CONFIG_FILES([ tests/zfs-tests/tests/functional/poolversion/Makefile tests/zfs-tests/tests/functional/privilege/Makefile tests/zfs-tests/tests/functional/quota/Makefile + tests/zfs-tests/tests/functional/raidz/Makefile tests/zfs-tests/tests/functional/redundancy/Makefile tests/zfs-tests/tests/functional/refquota/Makefile tests/zfs-tests/tests/functional/refreserv/Makefile diff --git a/copy-builtin b/copy-builtin index 762a34f9db18..716d2022c759 100755 --- a/copy-builtin +++ b/copy-builtin @@ -34,27 +34,8 @@ cp --recursive include "$KERNEL_DIR/include/zfs" cp --recursive module "$KERNEL_DIR/fs/zfs" cp zfs_config.h "$KERNEL_DIR/" -adjust_obj_paths() -{ - local FILE="$1" - local LINE OBJPATH - - while IFS='' read -r LINE - do - OBJPATH="${LINE#\$(MODULE)-objs += }" - if [ "$OBJPATH" = "$LINE" ] - then - echo "$LINE" - else - echo "\$(MODULE)-objs += ${OBJPATH##*/}" - fi - done < "$FILE" > "$FILE.new" - mv "$FILE.new" "$FILE" -} - for MODULE in "${MODULES[@]}" do - adjust_obj_paths "$KERNEL_DIR/fs/zfs/$MODULE/Makefile" sed -i.bak '/obj =/d' "$KERNEL_DIR/fs/zfs/$MODULE/Makefile" sed -i.bak '/src =/d' "$KERNEL_DIR/fs/zfs/$MODULE/Makefile" done diff --git a/etc/modules-load.d/zfs.conf.in b/etc/modules-load.d/zfs.conf.in index 73304bc2cd4a..8b41baa300cb 100644 --- a/etc/modules-load.d/zfs.conf.in +++ b/etc/modules-load.d/zfs.conf.in @@ -1 +1,3 @@ -zfs +# Always load kernel modules at boot. The default behavior is to load the +# kernel modules in the zfs-import-*.service or when blkid(8) detects a pool. +#zfs diff --git a/etc/systemd/system/50-zfs.preset.in b/etc/systemd/system/50-zfs.preset.in index 4efdd7200655..4d82778e35ba 100644 --- a/etc/systemd/system/50-zfs.preset.in +++ b/etc/systemd/system/50-zfs.preset.in @@ -1,2 +1,7 @@ # ZFS is enabled by default -enable zfs.* +enable zfs-import-cache.service +disable zfs-import-scan.service +enable zfs-mount.service +enable zfs-share.service +enable zfs-zed.service +enable zfs.target diff --git a/etc/systemd/system/zfs-import-cache.service.in b/etc/systemd/system/zfs-import-cache.service.in index a567b3095b91..02184a6027ff 100644 --- a/etc/systemd/system/zfs-import-cache.service.in +++ b/etc/systemd/system/zfs-import-cache.service.in @@ -4,6 +4,7 @@ DefaultDependencies=no Requires=systemd-udev-settle.service After=systemd-udev-settle.service After=cryptsetup.target +After=systemd-remount-fs.service Before=dracut-mount.service ConditionPathExists=@sysconfdir@/zfs/zpool.cache @@ -12,3 +13,7 @@ Type=oneshot RemainAfterExit=yes ExecStartPre=/sbin/modprobe zfs ExecStart=@sbindir@/zpool import -c @sysconfdir@/zfs/zpool.cache -aN + +[Install] +WantedBy=zfs-mount.service +WantedBy=zfs.target diff --git a/etc/systemd/system/zfs-import-scan.service.in b/etc/systemd/system/zfs-import-scan.service.in index cb3c2a7bdb1c..625f3a9553f8 100644 --- a/etc/systemd/system/zfs-import-scan.service.in +++ b/etc/systemd/system/zfs-import-scan.service.in @@ -11,4 +11,8 @@ ConditionPathExists=!@sysconfdir@/zfs/zpool.cache Type=oneshot RemainAfterExit=yes ExecStartPre=/sbin/modprobe zfs -ExecStart=@sbindir@/zpool import -d /dev/disk/by-id -aN +ExecStart=@sbindir@/zpool import -aN -o cachefile=none + +[Install] +WantedBy=zfs-mount.service +WantedBy=zfs.target diff --git a/etc/systemd/system/zfs-mount.service.in b/etc/systemd/system/zfs-mount.service.in index f1056af311cd..0664fd9e7665 100644 --- a/etc/systemd/system/zfs-mount.service.in +++ b/etc/systemd/system/zfs-mount.service.in @@ -1,15 +1,17 @@ [Unit] Description=Mount ZFS filesystems DefaultDependencies=no -Wants=zfs-import-cache.service -Wants=zfs-import-scan.service -Requires=systemd-udev-settle.service After=systemd-udev-settle.service After=zfs-import-cache.service After=zfs-import-scan.service +After=systemd-remount-fs.service Before=local-fs.target [Service] Type=oneshot RemainAfterExit=yes ExecStart=@sbindir@/zfs mount -a + +[Install] +WantedBy=zfs-share.service +WantedBy=zfs.target diff --git a/etc/systemd/system/zfs-share.service.in b/etc/systemd/system/zfs-share.service.in index 1230320e3381..688731ea3e27 100644 --- a/etc/systemd/system/zfs-share.service.in +++ b/etc/systemd/system/zfs-share.service.in @@ -3,12 +3,14 @@ Description=ZFS file system shares After=nfs-server.service nfs-kernel-server.service After=smb.service After=zfs-mount.service -Requires=zfs-mount.service PartOf=nfs-server.service nfs-kernel-server.service PartOf=smb.service [Service] Type=oneshot RemainAfterExit=yes -ExecStartPre=-@bindir@/rm /etc/dfs/sharetab +ExecStartPre=-@bindir@/rm -f /etc/dfs/sharetab ExecStart=@sbindir@/zfs share -a + +[Install] +WantedBy=zfs.target diff --git a/etc/systemd/system/zfs-zed.service.in b/etc/systemd/system/zfs-zed.service.in index 4d4c9d841323..e3dec3dca1b6 100644 --- a/etc/systemd/system/zfs-zed.service.in +++ b/etc/systemd/system/zfs-zed.service.in @@ -10,3 +10,4 @@ Restart=on-abort [Install] Alias=zed.service +WantedBy=zfs.target diff --git a/etc/systemd/system/zfs.target.in b/etc/systemd/system/zfs.target.in index ce110e573bac..4699463b0ddf 100644 --- a/etc/systemd/system/zfs.target.in +++ b/etc/systemd/system/zfs.target.in @@ -1,8 +1,5 @@ [Unit] Description=ZFS startup target -Requires=zfs-mount.service -Requires=zfs-share.service -Wants=zfs-zed.service [Install] WantedBy=multi-user.target diff --git a/include/libzfs.h b/include/libzfs.h index 654b93284318..f83e21423aa0 100644 --- a/include/libzfs.h +++ b/include/libzfs.h @@ -49,8 +49,6 @@ extern "C" { /* * Miscellaneous ZFS constants */ -#define ZFS_MAXNAMELEN MAXNAMELEN -#define ZPOOL_MAXNAMELEN MAXNAMELEN #define ZFS_MAXPROPLEN MAXPATHLEN #define ZPOOL_MAXPROPLEN MAXPATHLEN @@ -640,6 +638,10 @@ typedef boolean_t (snapfilter_cb_t)(zfs_handle_t *, void *); extern int zfs_send(zfs_handle_t *, const char *, const char *, sendflags_t *, int, snapfilter_cb_t, void *, nvlist_t **); extern int zfs_send_one(zfs_handle_t *, const char *, int, enum lzc_send_flags); +extern int zfs_send_resume(libzfs_handle_t *, sendflags_t *, int outfd, + const char *); +extern nvlist_t *zfs_send_resume_token_to_nvlist(libzfs_handle_t *hdl, + const char *token); extern int zfs_promote(zfs_handle_t *); extern int zfs_hold(zfs_handle_t *, const char *, const char *, @@ -680,6 +682,12 @@ typedef struct recvflags { /* set "canmount=off" on all modified filesystems */ boolean_t canmountoff; + /* + * Mark the file systems as "resumable" and do not destroy them if the + * receive is interrupted + */ + boolean_t resumable; + /* byteswap flag is used internally; callers need not specify */ boolean_t byteswap; diff --git a/include/libzfs_core.h b/include/libzfs_core.h index bdd6c951ee49..9e761004aa36 100644 --- a/include/libzfs_core.h +++ b/include/libzfs_core.h @@ -20,7 +20,7 @@ */ /* - * Copyright (c) 2013 by Delphix. All rights reserved. + * Copyright (c) 2012, 2014 by Delphix. All rights reserved. */ #ifndef _LIBZFS_CORE_H @@ -58,9 +58,21 @@ enum lzc_send_flags { }; int lzc_send(const char *, const char *, int, enum lzc_send_flags); -int lzc_receive(const char *, nvlist_t *, const char *, boolean_t, int); +int lzc_send_resume(const char *, const char *, int, + enum lzc_send_flags, uint64_t, uint64_t); int lzc_send_space(const char *, const char *, uint64_t *); +struct dmu_replay_record; + +int lzc_receive(const char *, nvlist_t *, const char *, boolean_t, int); +int lzc_receive_resumable(const char *, nvlist_t *, const char *, + boolean_t, int); +int lzc_receive_with_header(const char *, nvlist_t *, const char *, boolean_t, + boolean_t, int, const struct dmu_replay_record *); +int lzc_receive_one(const char *, nvlist_t *, const char *, boolean_t, + boolean_t, int, const struct dmu_replay_record *, int, uint64_t *, + uint64_t *, uint64_t *, nvlist_t **); + boolean_t lzc_exists(const char *); int lzc_rollback(const char *, char *, int); diff --git a/include/libzfs_impl.h b/include/libzfs_impl.h index ff02fa7949bd..36441ed7bb3a 100644 --- a/include/libzfs_impl.h +++ b/include/libzfs_impl.h @@ -21,17 +21,17 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2013 by Delphix. All rights reserved. + * Copyright (c) 2011, 2015 by Delphix. All rights reserved. */ #ifndef _LIBZFS_IMPL_H #define _LIBZFS_IMPL_H -#include #include -#include #include #include +#include +#include #include #include @@ -85,7 +85,7 @@ struct libzfs_handle { struct zfs_handle { libzfs_handle_t *zfs_hdl; zpool_handle_t *zpool_hdl; - char zfs_name[ZFS_MAXNAMELEN]; + char zfs_name[ZFS_MAX_DATASET_NAME_LEN]; zfs_type_t zfs_type; /* type including snapshot */ zfs_type_t zfs_head_type; /* type excluding snapshot */ dmu_objset_stats_t zfs_dmustats; @@ -106,7 +106,7 @@ struct zfs_handle { struct zpool_handle { libzfs_handle_t *zpool_hdl; zpool_handle_t *zpool_next; - char zpool_name[ZPOOL_MAXNAMELEN]; + char zpool_name[ZFS_MAX_DATASET_NAME_LEN]; int zpool_state; size_t zpool_config_size; nvlist_t *zpool_config; @@ -145,8 +145,6 @@ int zfs_standard_error_fmt(libzfs_handle_t *, int, const char *, ...); int zpool_standard_error(libzfs_handle_t *, int, const char *); int zpool_standard_error_fmt(libzfs_handle_t *, int, const char *, ...); -int get_dependents(libzfs_handle_t *, boolean_t, const char *, char ***, - size_t *); zfs_handle_t *make_dataset_handle_zc(libzfs_handle_t *, zfs_cmd_t *); zfs_handle_t *make_dataset_simple_handle_zc(zfs_handle_t *, zfs_cmd_t *); diff --git a/include/linux/blkdev_compat.h b/include/linux/blkdev_compat.h index 42b474b15f02..01bfad600ce2 100644 --- a/include/linux/blkdev_compat.h +++ b/include/linux/blkdev_compat.h @@ -305,12 +305,23 @@ bio_set_flags_failfast(struct block_device *bdev, int *flags) * The existence of these flags implies that REQ_FLUSH an REQ_FUA are * defined. Thus we can safely define VDEV_REQ_FLUSH and VDEV_REQ_FUA * compatibility macros. + * + * Linux 4.8 renamed the REQ_FLUSH to REQ_PREFLUSH but there was no + * functional change in behavior. */ #ifdef WRITE_FLUSH_FUA + #define VDEV_WRITE_FLUSH_FUA WRITE_FLUSH_FUA +#ifdef REQ_PREFLUSH +#define VDEV_REQ_FLUSH REQ_PREFLUSH +#define VDEV_REQ_FUA REQ_FUA +#else #define VDEV_REQ_FLUSH REQ_FLUSH #define VDEV_REQ_FUA REQ_FUA +#endif + #else + #define VDEV_WRITE_FLUSH_FUA WRITE_BARRIER #ifdef HAVE_BIO_RW_BARRIER #define VDEV_REQ_FLUSH (1 << BIO_RW_BARRIER) @@ -319,18 +330,34 @@ bio_set_flags_failfast(struct block_device *bdev, int *flags) #define VDEV_REQ_FLUSH REQ_HARDBARRIER #define VDEV_REQ_FUA REQ_FUA #endif + #endif /* - * 2.6.32 API change - * Use the normal I/O patch for discards. + * 2.6.28 - 2.6.35 API, + * BIO_RW_DISCARD + * + * 2.6.36 - 4.7 API, + * REQ_DISCARD + * + * 4.8 - 4.x API, + * REQ_OP_DISCARD + * + * In all cases the normal I/O path is used for discards. The only + * difference is how the kernel tags individual I/Os as discards. */ #ifdef QUEUE_FLAG_DISCARD -#ifdef HAVE_BIO_RW_DISCARD -#define VDEV_REQ_DISCARD (1 << BIO_RW_DISCARD) +static inline boolean_t +bio_is_discard(struct bio *bio) +{ +#if defined(HAVE_BIO_RW_DISCARD) + return (bio->bi_rw & (1 << BIO_RW_DISCARD)); +#elif defined(REQ_DISCARD) + return (bio->bi_rw & REQ_DISCARD); #else -#define VDEV_REQ_DISCARD REQ_DISCARD + return (bio_op(bio) == REQ_OP_DISCARD); #endif +} #else #error "Allowing the build will cause discard requests to become writes " "potentially triggering the DMU_MAX_ACCESS assertion. Please file a " diff --git a/include/linux/vfs_compat.h b/include/linux/vfs_compat.h index bcec1146a0c4..97220c2fe6b7 100644 --- a/include/linux/vfs_compat.h +++ b/include/linux/vfs_compat.h @@ -28,6 +28,7 @@ #define _ZFS_VFS_H #include +#include #include /* @@ -352,6 +353,58 @@ static inline struct inode *file_inode(const struct file *f) } #endif /* HAVE_FILE_INODE */ +#ifdef HAVE_KUID_HELPERS +static inline uid_t zfs_uid_read_impl(struct inode *ip) +{ + return (from_kuid(kcred->user_ns, ip->i_uid)); +} + +static inline uid_t zfs_uid_read(struct inode *ip) +{ + return (zfs_uid_read_impl(ip)); +} + +static inline gid_t zfs_gid_read_impl(struct inode *ip) +{ + return (from_kgid(kcred->user_ns, ip->i_gid)); +} + +static inline gid_t zfs_gid_read(struct inode *ip) +{ + return (zfs_gid_read_impl(ip)); +} + +static inline void zfs_uid_write(struct inode *ip, uid_t uid) +{ + ip->i_uid = make_kuid(kcred->user_ns, uid); +} + +static inline void zfs_gid_write(struct inode *ip, gid_t gid) +{ + ip->i_gid = make_kgid(kcred->user_ns, gid); +} +#else +static inline uid_t zfs_uid_read(struct inode *ip) +{ + return (ip->i_uid); +} + +static inline gid_t zfs_gid_read(struct inode *ip) +{ + return (ip->i_gid); +} + +static inline void zfs_uid_write(struct inode *ip, uid_t uid) +{ + ip->i_uid = uid; +} + +static inline void zfs_gid_write(struct inode *ip, gid_t gid) +{ + ip->i_gid = gid; +} +#endif + /* * 2.6.38 API change */ diff --git a/include/linux/xattr_compat.h b/include/linux/xattr_compat.h index 451b6545eb1f..b1c4293077c8 100644 --- a/include/linux/xattr_compat.h +++ b/include/linux/xattr_compat.h @@ -153,13 +153,27 @@ fn(struct inode *ip, const char *name, void *buffer, size_t size) \ } #endif +/* + * 4.7 API change, + * The xattr_handler->set() callback was changed to take a both dentry and + * inode, because the dentry might not be attached to an inode yet. + */ +#if defined(HAVE_XATTR_SET_DENTRY_INODE) +#define ZPL_XATTR_SET_WRAPPER(fn) \ +static int \ +fn(const struct xattr_handler *handler, struct dentry *dentry, \ + struct inode *inode, const char *name, const void *buffer, \ + size_t size, int flags) \ +{ \ + return (__ ## fn(inode, name, buffer, size, flags)); \ +} /* * 4.4 API change, * The xattr_handler->set() callback was changed to take a xattr_handler, * and handler_flags argument was removed and should be accessed by * handler->flags. */ -#if defined(HAVE_XATTR_SET_HANDLER) +#elif defined(HAVE_XATTR_SET_HANDLER) #define ZPL_XATTR_SET_WRAPPER(fn) \ static int \ fn(const struct xattr_handler *handler, struct dentry *dentry, \ diff --git a/include/sys/Makefile.am b/include/sys/Makefile.am index 98d7ad6fb061..9054597d1c03 100644 --- a/include/sys/Makefile.am +++ b/include/sys/Makefile.am @@ -1,4 +1,4 @@ -SUBDIRS = fm fs +SUBDIRS = fm fs crypto COMMON_H = \ $(top_srcdir)/include/sys/arc.h \ @@ -39,6 +39,7 @@ COMMON_H = \ $(top_srcdir)/include/sys/nvpair.h \ $(top_srcdir)/include/sys/nvpair_impl.h \ $(top_srcdir)/include/sys/pathname.h \ + $(top_srcdir)/include/sys/policy.h \ $(top_srcdir)/include/sys/range_tree.h \ $(top_srcdir)/include/sys/refcount.h \ $(top_srcdir)/include/sys/rrwlock.h \ @@ -50,9 +51,11 @@ COMMON_H = \ $(top_srcdir)/include/sys/space_reftree.h \ $(top_srcdir)/include/sys/spa.h \ $(top_srcdir)/include/sys/spa_impl.h \ + $(top_srcdir)/include/sys/spa_checksum.h \ $(top_srcdir)/include/sys/trace.h \ $(top_srcdir)/include/sys/trace_acl.h \ $(top_srcdir)/include/sys/trace_arc.h \ + $(top_srcdir)/include/sys/trace_common.h \ $(top_srcdir)/include/sys/trace_dbgmsg.h \ $(top_srcdir)/include/sys/trace_dbuf.h \ $(top_srcdir)/include/sys/trace_dmu.h \ @@ -60,6 +63,7 @@ COMMON_H = \ $(top_srcdir)/include/sys/trace_multilist.h \ $(top_srcdir)/include/sys/trace_txg.h \ $(top_srcdir)/include/sys/trace_zil.h \ + $(top_srcdir)/include/sys/trace_zio.h \ $(top_srcdir)/include/sys/trace_zrlock.h \ $(top_srcdir)/include/sys/txg.h \ $(top_srcdir)/include/sys/txg_impl.h \ @@ -74,6 +78,8 @@ COMMON_H = \ $(top_srcdir)/include/sys/vdev_file.h \ $(top_srcdir)/include/sys/vdev.h \ $(top_srcdir)/include/sys/vdev_impl.h \ + $(top_srcdir)/include/sys/vdev_raidz.h \ + $(top_srcdir)/include/sys/vdev_raidz_impl.h \ $(top_srcdir)/include/sys/xvattr.h \ $(top_srcdir)/include/sys/zap.h \ $(top_srcdir)/include/sys/zap_impl.h \ diff --git a/include/sys/arc.h b/include/sys/arc.h index 5780554b5c22..d8a85e830e15 100644 --- a/include/sys/arc.h +++ b/include/sys/arc.h @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */ @@ -134,7 +134,9 @@ typedef enum arc_space_type { ARC_SPACE_META, ARC_SPACE_HDRS, ARC_SPACE_L2HDRS, - ARC_SPACE_OTHER, + ARC_SPACE_DBUF, + ARC_SPACE_DNODE, + ARC_SPACE_BONUS, ARC_SPACE_NUMTYPES } arc_space_type_t; @@ -193,9 +195,11 @@ int arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, arc_flags_t *arc_flags, const zbookmark_phys_t *zb); zio_t *arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress, - const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone, - arc_done_func_t *done, void *private, zio_priority_t priority, - int zio_flags, const zbookmark_phys_t *zb); + const zio_prop_t *zp, + arc_done_func_t *ready, arc_done_func_t *child_ready, + arc_done_func_t *physdone, arc_done_func_t *done, + void *private, zio_priority_t priority, int zio_flags, + const zbookmark_phys_t *zb); arc_prune_t *arc_add_prune_callback(arc_prune_func_t *func, void *private); void arc_remove_prune_callback(arc_prune_t *p); diff --git a/include/sys/arc_impl.h b/include/sys/arc_impl.h index a9dbfc8dd73e..5c57c3157ae8 100644 --- a/include/sys/arc_impl.h +++ b/include/sys/arc_impl.h @@ -101,6 +101,7 @@ typedef struct arc_write_callback arc_write_callback_t; struct arc_write_callback { void *awcb_private; arc_done_func_t *awcb_ready; + arc_done_func_t *awcb_children_ready; arc_done_func_t *awcb_physdone; arc_done_func_t *awcb_done; arc_buf_t *awcb_buf; diff --git a/include/sys/crypto/Makefile.am b/include/sys/crypto/Makefile.am new file mode 100644 index 000000000000..7f8156b8f4a7 --- /dev/null +++ b/include/sys/crypto/Makefile.am @@ -0,0 +1,20 @@ +COMMON_H = \ + $(top_srcdir)/include/sys/crypto/api.h \ + $(top_srcdir)/include/sys/crypto/common.h \ + $(top_srcdir)/include/sys/crypto/icp.h + +KERNEL_H = + +USER_H = + +EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H) + +if CONFIG_USER +libzfsdir = $(includedir)/libzfs/sys/crypto +libzfs_HEADERS = $(COMMON_H) $(USER_H) +endif + +if CONFIG_KERNEL +kerneldir = @prefix@/src/zfs-$(VERSION)/include/sys/crypto +kernel_HEADERS = $(COMMON_H) $(KERNEL_H) +endif diff --git a/include/sys/crypto/api.h b/include/sys/crypto/api.h new file mode 100644 index 000000000000..7c3c465513de --- /dev/null +++ b/include/sys/crypto/api.h @@ -0,0 +1,425 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CRYPTO_API_H +#define _SYS_CRYPTO_API_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +typedef long crypto_req_id_t; +typedef void *crypto_bc_t; +typedef void *crypto_context_t; +typedef void *crypto_ctx_template_t; + +typedef uint32_t crypto_call_flag_t; + +/* crypto_call_flag's values */ +#define CRYPTO_ALWAYS_QUEUE 0x00000001 /* ALWAYS queue the req. */ +#define CRYPTO_NOTIFY_OPDONE 0x00000002 /* Notify intermediate steps */ +#define CRYPTO_SKIP_REQID 0x00000004 /* Skip request ID generation */ +#define CRYPTO_RESTRICTED 0x00000008 /* cannot use restricted prov */ + +typedef struct { + crypto_call_flag_t cr_flag; + void (*cr_callback_func)(void *, int); + void *cr_callback_arg; + crypto_req_id_t cr_reqid; +} crypto_call_req_t; + +/* + * Returns the mechanism type corresponding to a mechanism name. + */ + +#define CRYPTO_MECH_INVALID ((uint64_t)-1) +extern crypto_mech_type_t crypto_mech2id(crypto_mech_name_t name); + +/* + * Create and destroy context templates. + */ +extern int crypto_create_ctx_template(crypto_mechanism_t *mech, + crypto_key_t *key, crypto_ctx_template_t *tmpl, int kmflag); +extern void crypto_destroy_ctx_template(crypto_ctx_template_t tmpl); + +/* + * Single and multi-part digest operations. + */ +extern int crypto_digest(crypto_mechanism_t *mech, crypto_data_t *data, + crypto_data_t *digest, crypto_call_req_t *cr); +extern int crypto_digest_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_data_t *, crypto_data_t *, + crypto_call_req_t *); +extern int crypto_digest_init(crypto_mechanism_t *mech, crypto_context_t *ctxp, + crypto_call_req_t *cr); +extern int crypto_digest_init_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_context_t *, crypto_call_req_t *); +extern int crypto_digest_update(crypto_context_t ctx, crypto_data_t *data, + crypto_call_req_t *cr); +extern int crypto_digest_final(crypto_context_t ctx, crypto_data_t *digest, + crypto_call_req_t *cr); + +/* + * Single and multi-part MAC operations. + */ +extern int crypto_mac(crypto_mechanism_t *mech, crypto_data_t *data, + crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac, + crypto_call_req_t *cr); +extern int crypto_mac_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_data_t *, crypto_key_t *, + crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); +extern int crypto_mac_verify(crypto_mechanism_t *mech, crypto_data_t *data, + crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac, + crypto_call_req_t *cr); +extern int crypto_mac_verify_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_data_t *, crypto_key_t *, + crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); +extern int crypto_mac_init(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); +extern int crypto_mac_init_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, + crypto_context_t *, crypto_call_req_t *); +extern int crypto_mac_update(crypto_context_t ctx, crypto_data_t *data, + crypto_call_req_t *cr); +extern int crypto_mac_final(crypto_context_t ctx, crypto_data_t *data, + crypto_call_req_t *cr); + +/* + * Single and multi-part sign with private key operations. + */ +extern int crypto_sign(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_data_t *data, crypto_ctx_template_t tmpl, + crypto_data_t *signature, crypto_call_req_t *cr); +extern int crypto_sign_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); +extern int crypto_sign_init(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); +extern int crypto_sign_init_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, + crypto_context_t *, crypto_call_req_t *); +extern int crypto_sign_update(crypto_context_t ctx, crypto_data_t *data, + crypto_call_req_t *cr); +extern int crypto_sign_final(crypto_context_t ctx, crypto_data_t *signature, + crypto_call_req_t *cr); +extern int crypto_sign_recover_init_prov(crypto_provider_t, + crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, + crypto_ctx_template_t tmpl, crypto_context_t *, crypto_call_req_t *); +extern int crypto_sign_recover(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_data_t *data, crypto_ctx_template_t tmpl, crypto_data_t *signature, + crypto_call_req_t *cr); +extern int crypto_sign_recover_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); + +/* + * Single and multi-part verify with public key operations. + */ +extern int crypto_verify(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_data_t *data, crypto_ctx_template_t tmpl, crypto_data_t *signature, + crypto_call_req_t *cr); +extern int crypto_verify_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); +extern int crypto_verify_init(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); +extern int crypto_verify_init_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, + crypto_context_t *, crypto_call_req_t *); +extern int crypto_verify_update(crypto_context_t ctx, crypto_data_t *data, + crypto_call_req_t *cr); +extern int crypto_verify_final(crypto_context_t ctx, crypto_data_t *signature, + crypto_call_req_t *cr); +extern int crypto_verify_recover_init_prov(crypto_provider_t, + crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, + crypto_ctx_template_t tmpl, crypto_context_t *, crypto_call_req_t *); +extern int crypto_verify_recover(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_data_t *signature, crypto_ctx_template_t tmpl, crypto_data_t *data, + crypto_call_req_t *cr); +extern int crypto_verify_recover_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); + +/* + * Single and multi-part encryption operations. + */ +extern int crypto_encrypt(crypto_mechanism_t *mech, crypto_data_t *plaintext, + crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *ciphertext, + crypto_call_req_t *cr); +extern int crypto_encrypt_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_data_t *, crypto_key_t *, + crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); +extern int crypto_encrypt_init(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, crypto_call_req_t *cr); +extern int crypto_encrypt_init_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, + crypto_context_t *, crypto_call_req_t *); +extern int crypto_encrypt_update(crypto_context_t ctx, + crypto_data_t *plaintext, crypto_data_t *ciphertext, + crypto_call_req_t *cr); +extern int crypto_encrypt_final(crypto_context_t ctx, + crypto_data_t *ciphertext, crypto_call_req_t *cr); + +/* + * Single and multi-part decryption operations. + */ +extern int crypto_decrypt(crypto_mechanism_t *mech, crypto_data_t *ciphertext, + crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *plaintext, + crypto_call_req_t *cr); +extern int crypto_decrypt_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_data_t *, crypto_key_t *, + crypto_ctx_template_t, crypto_data_t *, crypto_call_req_t *); +extern int crypto_decrypt_init(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, + crypto_call_req_t *cr); +extern int crypto_decrypt_init_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_ctx_template_t, + crypto_context_t *, crypto_call_req_t *); +extern int crypto_decrypt_update(crypto_context_t ctx, + crypto_data_t *ciphertext, crypto_data_t *plaintext, + crypto_call_req_t *cr); +extern int crypto_decrypt_final(crypto_context_t ctx, crypto_data_t *plaintext, + crypto_call_req_t *cr); + +/* + * Single and multi-part encrypt/MAC dual operations. + */ +extern int crypto_encrypt_mac(crypto_mechanism_t *encr_mech, + crypto_mechanism_t *mac_mech, crypto_data_t *pt, + crypto_key_t *encr_key, crypto_key_t *mac_key, + crypto_ctx_template_t encr_tmpl, crypto_ctx_template_t mac_tmpl, + crypto_dual_data_t *ct, crypto_data_t *mac, crypto_call_req_t *cr); +extern int crypto_encrypt_mac_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_mechanism_t *, crypto_data_t *, + crypto_key_t *, crypto_key_t *, crypto_ctx_template_t, + crypto_ctx_template_t, crypto_dual_data_t *, crypto_data_t *, + crypto_call_req_t *); +extern int crypto_encrypt_mac_init(crypto_mechanism_t *encr_mech, + crypto_mechanism_t *mac_mech, crypto_key_t *encr_key, + crypto_key_t *mac_key, crypto_ctx_template_t encr_tmpl, + crypto_ctx_template_t mac_tmpl, crypto_context_t *ctxp, + crypto_call_req_t *cr); +extern int crypto_encrypt_mac_init_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_mechanism_t *, crypto_key_t *, crypto_key_t *, + crypto_ctx_template_t, crypto_ctx_template_t, crypto_context_t *, + crypto_call_req_t *); +extern int crypto_encrypt_mac_update(crypto_context_t ctx, + crypto_data_t *pt, crypto_dual_data_t *ct, crypto_call_req_t *cr); +extern int crypto_encrypt_mac_final(crypto_context_t ctx, + crypto_dual_data_t *ct, crypto_data_t *mac, crypto_call_req_t *cr); + +/* + * Single and multi-part MAC/decrypt dual operations. + */ +extern int crypto_mac_decrypt(crypto_mechanism_t *mac_mech, + crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct, + crypto_key_t *mac_key, crypto_key_t *decr_key, + crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, + crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr); +extern int crypto_mac_decrypt_prov(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *mac_mech, crypto_mechanism_t *decr_mech, + crypto_dual_data_t *ct, crypto_key_t *mac_key, crypto_key_t *decr_key, + crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, + crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr); +extern int crypto_mac_verify_decrypt(crypto_mechanism_t *mac_mech, + crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct, + crypto_key_t *mac_key, crypto_key_t *decr_key, + crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, + crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr); +extern int crypto_mac_verify_decrypt_prov(crypto_provider_t, + crypto_session_id_t, crypto_mechanism_t *mac_mech, + crypto_mechanism_t *decr_mech, crypto_dual_data_t *ct, + crypto_key_t *mac_key, crypto_key_t *decr_key, + crypto_ctx_template_t mac_tmpl, crypto_ctx_template_t decr_tmpl, + crypto_data_t *mac, crypto_data_t *pt, crypto_call_req_t *cr); +extern int crypto_mac_decrypt_init(crypto_mechanism_t *mac_mech, + crypto_mechanism_t *decr_mech, crypto_key_t *mac_key, + crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl, + crypto_ctx_template_t decr_tmpl, crypto_context_t *ctxp, + crypto_call_req_t *cr); +extern int crypto_mac_decrypt_init_prov(crypto_provider_t, + crypto_session_id_t, crypto_mechanism_t *mac_mech, + crypto_mechanism_t *decr_mech, crypto_key_t *mac_key, + crypto_key_t *decr_key, crypto_ctx_template_t mac_tmpl, + crypto_ctx_template_t decr_tmpl, crypto_context_t *ctxp, + crypto_call_req_t *cr); +extern int crypto_mac_decrypt_update(crypto_context_t ctx, + crypto_dual_data_t *ct, crypto_data_t *pt, crypto_call_req_t *cr); +extern int crypto_mac_decrypt_final(crypto_context_t ctx, crypto_data_t *mac, + crypto_data_t *pt, crypto_call_req_t *cr); + +/* Session Management */ +extern int crypto_session_open(crypto_provider_t, crypto_session_id_t *, + crypto_call_req_t *); +extern int crypto_session_close(crypto_provider_t, crypto_session_id_t, + crypto_call_req_t *); +extern int crypto_session_login(crypto_provider_t, crypto_session_id_t, + crypto_user_type_t, char *, size_t, crypto_call_req_t *); +extern int crypto_session_logout(crypto_provider_t, crypto_session_id_t, + crypto_call_req_t *); + +/* Object Management */ +extern int crypto_object_copy(crypto_provider_t, crypto_session_id_t, + crypto_object_id_t, crypto_object_attribute_t *, uint_t, + crypto_object_id_t *, crypto_call_req_t *); +extern int crypto_object_create(crypto_provider_t, crypto_session_id_t, + crypto_object_attribute_t *, uint_t, crypto_object_id_t *, + crypto_call_req_t *); +extern int crypto_object_destroy(crypto_provider_t, crypto_session_id_t, + crypto_object_id_t, crypto_call_req_t *); +extern int crypto_object_get_attribute_value(crypto_provider_t, + crypto_session_id_t, crypto_object_id_t, crypto_object_attribute_t *, + uint_t, crypto_call_req_t *); +extern int crypto_object_get_size(crypto_provider_t, crypto_session_id_t, + crypto_object_id_t, size_t *, crypto_call_req_t *); +extern int crypto_object_find_final(crypto_provider_t, void *, + crypto_call_req_t *); +extern int crypto_object_find_init(crypto_provider_t, crypto_session_id_t, + crypto_object_attribute_t *, uint_t, void **, crypto_call_req_t *); +extern int crypto_object_find(crypto_provider_t, void *, crypto_object_id_t *, + uint_t *, uint_t, crypto_call_req_t *); +extern int crypto_object_set_attribute_value(crypto_provider_t, + crypto_session_id_t, crypto_object_id_t, crypto_object_attribute_t *, + uint_t, crypto_call_req_t *); + +/* Key Management */ +extern int crypto_key_derive(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_object_attribute_t *, + uint_t, crypto_object_id_t *, crypto_call_req_t *); +extern int crypto_key_generate(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_object_attribute_t *, uint_t, + crypto_object_id_t *, crypto_call_req_t *); +extern int crypto_key_generate_pair(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_object_attribute_t *, uint_t, + crypto_object_attribute_t *, uint_t, crypto_object_id_t *, + crypto_object_id_t *, crypto_call_req_t *); +extern int crypto_key_unwrap(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, uchar_t *, size_t *, + crypto_object_attribute_t *, uint_t, crypto_object_id_t *, + crypto_call_req_t *); +extern int crypto_key_wrap(crypto_provider_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_object_id_t *, uchar_t *, + size_t *, crypto_call_req_t *); +extern int crypto_key_check_prov(crypto_provider_t, crypto_mechanism_t *mech, + crypto_key_t *key); +extern int crypto_key_check(crypto_mechanism_t *mech, crypto_key_t *key); + + +/* + * Routines to cancel a single asynchronous request or all asynchronous + * requests associated with a particular context. + */ +extern void crypto_cancel_req(crypto_req_id_t req); +extern void crypto_cancel_ctx(crypto_context_t ctx); + +/* + * crypto_get_mech_list(9F) allocates and returns the list of currently + * supported cryptographic mechanisms. + */ +extern crypto_mech_name_t *crypto_get_mech_list(uint_t *count, int kmflag); +extern void crypto_free_mech_list(crypto_mech_name_t *mech_names, + uint_t count); + +extern crypto_provider_t crypto_get_provider(char *, char *, char *); +extern int crypto_get_provinfo(crypto_provider_t, crypto_provider_ext_info_t *); +extern void crypto_release_provider(crypto_provider_t); + +/* + * A kernel consumer can request to be notified when some particular event + * occurs. The valid events, callback function type, and functions to + * be called to register or unregister for notification are defined below. + */ + +#define CRYPTO_EVENT_MECHS_CHANGED 0x00000001 +#define CRYPTO_EVENT_PROVIDER_REGISTERED 0x00000002 +#define CRYPTO_EVENT_PROVIDER_UNREGISTERED 0x00000004 + +typedef enum { + CRYPTO_MECH_ADDED = 1, + CRYPTO_MECH_REMOVED +} crypto_event_change_t; + +/* The event_arg argument structure for CRYPTO_EVENT_PROVIDERS_CHANGE event */ +typedef struct crypto_notify_event_change { + crypto_mech_name_t ec_mech_name; + crypto_provider_type_t ec_provider_type; + crypto_event_change_t ec_change; +} crypto_notify_event_change_t; + +typedef void *crypto_notify_handle_t; +typedef void (*crypto_notify_callback_t)(uint32_t event_mask, void *event_arg); + +extern crypto_notify_handle_t crypto_notify_events( + crypto_notify_callback_t nf, uint32_t event_mask); +extern void crypto_unnotify_events(crypto_notify_handle_t); + +/* + * crypto_bufcall(9F) group of routines. + */ +extern crypto_bc_t crypto_bufcall_alloc(void); +extern int crypto_bufcall_free(crypto_bc_t bc); +extern int crypto_bufcall(crypto_bc_t bc, void (*func)(void *arg), void *arg); +extern int crypto_unbufcall(crypto_bc_t bc); + +/* + * To obtain the list of key size ranges supported by a mechanism. + */ + +#define CRYPTO_MECH_USAGE_ENCRYPT 0x00000001 +#define CRYPTO_MECH_USAGE_DECRYPT 0x00000002 +#define CRYPTO_MECH_USAGE_MAC 0x00000004 + +typedef uint32_t crypto_mech_usage_t; + +typedef struct crypto_mechanism_info { + size_t mi_min_key_size; + size_t mi_max_key_size; + crypto_keysize_unit_t mi_keysize_unit; /* for mi_xxx_key_size */ + crypto_mech_usage_t mi_usage; +} crypto_mechanism_info_t; + +#ifdef _SYSCALL32 + +typedef struct crypto_mechanism_info32 { + size32_t mi_min_key_size; + size32_t mi_max_key_size; + crypto_keysize_unit_t mi_keysize_unit; /* for mi_xxx_key_size */ + crypto_mech_usage_t mi_usage; +} crypto_mechanism_info32_t; + +#endif /* _SYSCALL32 */ + +extern int crypto_get_all_mech_info(crypto_mech_type_t, + crypto_mechanism_info_t **, uint_t *, int); +extern void crypto_free_all_mech_info(crypto_mechanism_info_t *, uint_t); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRYPTO_API_H */ diff --git a/include/sys/crypto/common.h b/include/sys/crypto/common.h new file mode 100644 index 000000000000..a4f9d9848c23 --- /dev/null +++ b/include/sys/crypto/common.h @@ -0,0 +1,583 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + */ +/* + * Copyright 2013 Saso Kiselkov. All rights reserved. + */ + +#ifndef _SYS_CRYPTO_COMMON_H +#define _SYS_CRYPTO_COMMON_H + +/* + * Header file for the common data structures of the cryptographic framework + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* Cryptographic Mechanisms */ + +#define CRYPTO_MAX_MECH_NAME 32 +typedef char crypto_mech_name_t[CRYPTO_MAX_MECH_NAME]; + +typedef uint64_t crypto_mech_type_t; + +typedef struct crypto_mechanism { + crypto_mech_type_t cm_type; /* mechanism type */ + caddr_t cm_param; /* mech. parameter */ + size_t cm_param_len; /* mech. parameter len */ +} crypto_mechanism_t; + +#ifdef _SYSCALL32 + +typedef struct crypto_mechanism32 { + crypto_mech_type_t cm_type; /* mechanism type */ + caddr32_t cm_param; /* mech. parameter */ + size32_t cm_param_len; /* mech. parameter len */ +} crypto_mechanism32_t; + +#endif /* _SYSCALL32 */ + +/* CK_AES_CTR_PARAMS provides parameters to the CKM_AES_CTR mechanism */ +typedef struct CK_AES_CTR_PARAMS { + ulong_t ulCounterBits; + uint8_t cb[16]; +} CK_AES_CTR_PARAMS; + +/* CK_AES_CCM_PARAMS provides parameters to the CKM_AES_CCM mechanism */ +typedef struct CK_AES_CCM_PARAMS { + ulong_t ulMACSize; + ulong_t ulNonceSize; + ulong_t ulAuthDataSize; + ulong_t ulDataSize; /* used for plaintext or ciphertext */ + uchar_t *nonce; + uchar_t *authData; +} CK_AES_CCM_PARAMS; + +/* CK_AES_GCM_PARAMS provides parameters to the CKM_AES_GCM mechanism */ +typedef struct CK_AES_GCM_PARAMS { + uchar_t *pIv; + ulong_t ulIvLen; + ulong_t ulIvBits; + uchar_t *pAAD; + ulong_t ulAADLen; + ulong_t ulTagBits; +} CK_AES_GCM_PARAMS; + +/* CK_AES_GMAC_PARAMS provides parameters to the CKM_AES_GMAC mechanism */ +typedef struct CK_AES_GMAC_PARAMS { + uchar_t *pIv; + uchar_t *pAAD; + ulong_t ulAADLen; +} CK_AES_GMAC_PARAMS; + +/* + * CK_ECDH1_DERIVE_PARAMS provides the parameters to the + * CKM_ECDH1_KEY_DERIVE mechanism + */ +typedef struct CK_ECDH1_DERIVE_PARAMS { + ulong_t kdf; + ulong_t ulSharedDataLen; + uchar_t *pSharedData; + ulong_t ulPublicDataLen; + uchar_t *pPublicData; +} CK_ECDH1_DERIVE_PARAMS; + +#ifdef _SYSCALL32 + +/* needed for 32-bit applications running on 64-bit kernels */ +typedef struct CK_AES_CTR_PARAMS32 { + uint32_t ulCounterBits; + uint8_t cb[16]; +} CK_AES_CTR_PARAMS32; + +/* needed for 32-bit applications running on 64-bit kernels */ +typedef struct CK_AES_CCM_PARAMS32 { + uint32_t ulMACSize; + uint32_t ulNonceSize; + uint32_t ulAuthDataSize; + uint32_t ulDataSize; + caddr32_t nonce; + caddr32_t authData; +} CK_AES_CCM_PARAMS32; + +/* needed for 32-bit applications running on 64-bit kernels */ +typedef struct CK_AES_GCM_PARAMS32 { + caddr32_t pIv; + uint32_t ulIvLen; + uint32_t ulIvBits; + caddr32_t pAAD; + uint32_t ulAADLen; + uint32_t ulTagBits; +} CK_AES_GCM_PARAMS32; + +/* needed for 32-bit applications running on 64-bit kernels */ +typedef struct CK_AES_GMAC_PARAMS32 { + caddr32_t pIv; + caddr32_t pAAD; + uint32_t ulAADLen; +} CK_AES_GMAC_PARAMS32; + +typedef struct CK_ECDH1_DERIVE_PARAMS32 { + uint32_t kdf; + uint32_t ulSharedDataLen; + caddr32_t pSharedData; + uint32_t ulPublicDataLen; + caddr32_t pPublicData; +} CK_ECDH1_DERIVE_PARAMS32; + +#endif /* _SYSCALL32 */ + +/* + * The measurement unit bit flag for a mechanism's minimum or maximum key size. + * The unit are mechanism dependent. It can be in bits or in bytes. + */ +typedef uint32_t crypto_keysize_unit_t; + +/* + * The following bit flags are valid in cm_mech_flags field in + * the crypto_mech_info_t structure of the SPI. + * + * Only the first two bit flags are valid in mi_keysize_unit + * field in the crypto_mechanism_info_t structure of the API. + */ +#define CRYPTO_KEYSIZE_UNIT_IN_BITS 0x00000001 +#define CRYPTO_KEYSIZE_UNIT_IN_BYTES 0x00000002 +#define CRYPTO_CAN_SHARE_OPSTATE 0x00000004 /* supports sharing */ + + +/* Mechanisms supported out-of-the-box */ +#define SUN_CKM_MD4 "CKM_MD4" +#define SUN_CKM_MD5 "CKM_MD5" +#define SUN_CKM_MD5_HMAC "CKM_MD5_HMAC" +#define SUN_CKM_MD5_HMAC_GENERAL "CKM_MD5_HMAC_GENERAL" +#define SUN_CKM_SHA1 "CKM_SHA_1" +#define SUN_CKM_SHA1_HMAC "CKM_SHA_1_HMAC" +#define SUN_CKM_SHA1_HMAC_GENERAL "CKM_SHA_1_HMAC_GENERAL" +#define SUN_CKM_SHA256 "CKM_SHA256" +#define SUN_CKM_SHA256_HMAC "CKM_SHA256_HMAC" +#define SUN_CKM_SHA256_HMAC_GENERAL "CKM_SHA256_HMAC_GENERAL" +#define SUN_CKM_SHA384 "CKM_SHA384" +#define SUN_CKM_SHA384_HMAC "CKM_SHA384_HMAC" +#define SUN_CKM_SHA384_HMAC_GENERAL "CKM_SHA384_HMAC_GENERAL" +#define SUN_CKM_SHA512 "CKM_SHA512" +#define SUN_CKM_SHA512_HMAC "CKM_SHA512_HMAC" +#define SUN_CKM_SHA512_HMAC_GENERAL "CKM_SHA512_HMAC_GENERAL" +#define SUN_CKM_SHA512_224 "CKM_SHA512_224" +#define SUN_CKM_SHA512_256 "CKM_SHA512_256" +#define SUN_CKM_DES_CBC "CKM_DES_CBC" +#define SUN_CKM_DES3_CBC "CKM_DES3_CBC" +#define SUN_CKM_DES_ECB "CKM_DES_ECB" +#define SUN_CKM_DES3_ECB "CKM_DES3_ECB" +#define SUN_CKM_BLOWFISH_CBC "CKM_BLOWFISH_CBC" +#define SUN_CKM_BLOWFISH_ECB "CKM_BLOWFISH_ECB" +#define SUN_CKM_AES_CBC "CKM_AES_CBC" +#define SUN_CKM_AES_ECB "CKM_AES_ECB" +#define SUN_CKM_AES_CTR "CKM_AES_CTR" +#define SUN_CKM_AES_CCM "CKM_AES_CCM" +#define SUN_CKM_AES_GCM "CKM_AES_GCM" +#define SUN_CKM_AES_GMAC "CKM_AES_GMAC" +#define SUN_CKM_AES_CFB128 "CKM_AES_CFB128" +#define SUN_CKM_RC4 "CKM_RC4" +#define SUN_CKM_RSA_PKCS "CKM_RSA_PKCS" +#define SUN_CKM_RSA_X_509 "CKM_RSA_X_509" +#define SUN_CKM_MD5_RSA_PKCS "CKM_MD5_RSA_PKCS" +#define SUN_CKM_SHA1_RSA_PKCS "CKM_SHA1_RSA_PKCS" +#define SUN_CKM_SHA256_RSA_PKCS "CKM_SHA256_RSA_PKCS" +#define SUN_CKM_SHA384_RSA_PKCS "CKM_SHA384_RSA_PKCS" +#define SUN_CKM_SHA512_RSA_PKCS "CKM_SHA512_RSA_PKCS" +#define SUN_CKM_EC_KEY_PAIR_GEN "CKM_EC_KEY_PAIR_GEN" +#define SUN_CKM_ECDH1_DERIVE "CKM_ECDH1_DERIVE" +#define SUN_CKM_ECDSA_SHA1 "CKM_ECDSA_SHA1" +#define SUN_CKM_ECDSA "CKM_ECDSA" + +/* Shared operation context format for CKM_RC4 */ +typedef struct { +#if defined(__amd64) + uint32_t i, j; + uint32_t arr[256]; + uint32_t flag; +#else + uchar_t arr[256]; + uchar_t i, j; +#endif /* __amd64 */ + uint64_t pad; /* For 64-bit alignment */ +} arcfour_state_t; + +/* Data arguments of cryptographic operations */ + +typedef enum crypto_data_format { + CRYPTO_DATA_RAW = 1, + CRYPTO_DATA_UIO, +} crypto_data_format_t; + +typedef struct crypto_data { + crypto_data_format_t cd_format; /* Format identifier */ + off_t cd_offset; /* Offset from the beginning */ + size_t cd_length; /* # of bytes in use */ + caddr_t cd_miscdata; /* ancillary data */ + union { + /* Raw format */ + iovec_t cdu_raw; /* Pointer and length */ + + /* uio scatter-gather format */ + uio_t *cdu_uio; + + } cdu; /* Crypto Data Union */ +} crypto_data_t; + +#define cd_raw cdu.cdu_raw +#define cd_uio cdu.cdu_uio +#define cd_mp cdu.cdu_mp + +typedef struct crypto_dual_data { + crypto_data_t dd_data; /* The data */ + off_t dd_offset2; /* Used by dual operation */ + size_t dd_len2; /* # of bytes to take */ +} crypto_dual_data_t; + +#define dd_format dd_data.cd_format +#define dd_offset1 dd_data.cd_offset +#define dd_len1 dd_data.cd_length +#define dd_miscdata dd_data.cd_miscdata +#define dd_raw dd_data.cd_raw +#define dd_uio dd_data.cd_uio +#define dd_mp dd_data.cd_mp + +/* The keys, and their contents */ + +typedef enum { + CRYPTO_KEY_RAW = 1, /* ck_data is a cleartext key */ + CRYPTO_KEY_REFERENCE, /* ck_obj_id is an opaque reference */ + CRYPTO_KEY_ATTR_LIST /* ck_attrs is a list of object attributes */ +} crypto_key_format_t; + +typedef uint64_t crypto_attr_type_t; + +/* Attribute types to use for passing a RSA public key or a private key. */ +#define SUN_CKA_MODULUS 0x00000120 +#define SUN_CKA_MODULUS_BITS 0x00000121 +#define SUN_CKA_PUBLIC_EXPONENT 0x00000122 +#define SUN_CKA_PRIVATE_EXPONENT 0x00000123 +#define SUN_CKA_PRIME_1 0x00000124 +#define SUN_CKA_PRIME_2 0x00000125 +#define SUN_CKA_EXPONENT_1 0x00000126 +#define SUN_CKA_EXPONENT_2 0x00000127 +#define SUN_CKA_COEFFICIENT 0x00000128 +#define SUN_CKA_PRIME 0x00000130 +#define SUN_CKA_SUBPRIME 0x00000131 +#define SUN_CKA_BASE 0x00000132 + +#define CKK_EC 0x00000003 +#define CKK_GENERIC_SECRET 0x00000010 +#define CKK_RC4 0x00000012 +#define CKK_AES 0x0000001F +#define CKK_DES 0x00000013 +#define CKK_DES2 0x00000014 +#define CKK_DES3 0x00000015 + +#define CKO_PUBLIC_KEY 0x00000002 +#define CKO_PRIVATE_KEY 0x00000003 +#define CKA_CLASS 0x00000000 +#define CKA_VALUE 0x00000011 +#define CKA_KEY_TYPE 0x00000100 +#define CKA_VALUE_LEN 0x00000161 +#define CKA_EC_PARAMS 0x00000180 +#define CKA_EC_POINT 0x00000181 + +typedef uint32_t crypto_object_id_t; + +typedef struct crypto_object_attribute { + crypto_attr_type_t oa_type; /* attribute type */ + caddr_t oa_value; /* attribute value */ + ssize_t oa_value_len; /* length of attribute value */ +} crypto_object_attribute_t; + +typedef struct crypto_key { + crypto_key_format_t ck_format; /* format identifier */ + union { + /* for CRYPTO_KEY_RAW ck_format */ + struct { + uint_t cku_v_length; /* # of bits in ck_data */ + void *cku_v_data; /* ptr to key value */ + } cku_key_value; + + /* for CRYPTO_KEY_REFERENCE ck_format */ + crypto_object_id_t cku_key_id; /* reference to object key */ + + /* for CRYPTO_KEY_ATTR_LIST ck_format */ + struct { + uint_t cku_a_count; /* number of attributes */ + crypto_object_attribute_t *cku_a_oattr; + } cku_key_attrs; + } cku_data; /* Crypto Key union */ +} crypto_key_t; + +#ifdef _SYSCALL32 + +typedef struct crypto_object_attribute32 { + uint64_t oa_type; /* attribute type */ + caddr32_t oa_value; /* attribute value */ + ssize32_t oa_value_len; /* length of attribute value */ +} crypto_object_attribute32_t; + +typedef struct crypto_key32 { + crypto_key_format_t ck_format; /* format identifier */ + union { + /* for CRYPTO_KEY_RAW ck_format */ + struct { + uint32_t cku_v_length; /* # of bytes in ck_data */ + caddr32_t cku_v_data; /* ptr to key value */ + } cku_key_value; + + /* for CRYPTO_KEY_REFERENCE ck_format */ + crypto_object_id_t cku_key_id; /* reference to object key */ + + /* for CRYPTO_KEY_ATTR_LIST ck_format */ + struct { + uint32_t cku_a_count; /* number of attributes */ + caddr32_t cku_a_oattr; + } cku_key_attrs; + } cku_data; /* Crypto Key union */ +} crypto_key32_t; + +#endif /* _SYSCALL32 */ + +#define ck_data cku_data.cku_key_value.cku_v_data +#define ck_length cku_data.cku_key_value.cku_v_length +#define ck_obj_id cku_data.cku_key_id +#define ck_count cku_data.cku_key_attrs.cku_a_count +#define ck_attrs cku_data.cku_key_attrs.cku_a_oattr + +/* + * Raw key lengths are expressed in number of bits. + * The following macro returns the minimum number of + * bytes that can contain the specified number of bits. + * Round up without overflowing the integer type. + */ +#define CRYPTO_BITS2BYTES(n) ((n) == 0 ? 0 : (((n) - 1) >> 3) + 1) +#define CRYPTO_BYTES2BITS(n) ((n) << 3) + +/* Providers */ + +typedef enum { + CRYPTO_HW_PROVIDER = 0, + CRYPTO_SW_PROVIDER, + CRYPTO_LOGICAL_PROVIDER +} crypto_provider_type_t; + +typedef uint32_t crypto_provider_id_t; +#define KCF_PROVID_INVALID ((uint32_t)-1) + +typedef struct crypto_provider_entry { + crypto_provider_id_t pe_provider_id; + uint_t pe_mechanism_count; +} crypto_provider_entry_t; + +typedef struct crypto_dev_list_entry { + char le_dev_name[MAXNAMELEN]; + uint_t le_dev_instance; + uint_t le_mechanism_count; +} crypto_dev_list_entry_t; + +/* User type for authentication ioctls and SPI entry points */ + +typedef enum crypto_user_type { + CRYPTO_SO = 0, + CRYPTO_USER +} crypto_user_type_t; + +/* Version for provider management ioctls and SPI entry points */ + +typedef struct crypto_version { + uchar_t cv_major; + uchar_t cv_minor; +} crypto_version_t; + +/* session data structure opaque to the consumer */ +typedef void *crypto_session_t; + +/* provider data structure opaque to the consumer */ +typedef void *crypto_provider_t; + +/* Limits used by both consumers and providers */ +#define CRYPTO_EXT_SIZE_LABEL 32 +#define CRYPTO_EXT_SIZE_MANUF 32 +#define CRYPTO_EXT_SIZE_MODEL 16 +#define CRYPTO_EXT_SIZE_SERIAL 16 +#define CRYPTO_EXT_SIZE_TIME 16 + +typedef struct crypto_provider_ext_info { + uchar_t ei_label[CRYPTO_EXT_SIZE_LABEL]; + uchar_t ei_manufacturerID[CRYPTO_EXT_SIZE_MANUF]; + uchar_t ei_model[CRYPTO_EXT_SIZE_MODEL]; + uchar_t ei_serial_number[CRYPTO_EXT_SIZE_SERIAL]; + ulong_t ei_flags; + ulong_t ei_max_session_count; + ulong_t ei_max_pin_len; + ulong_t ei_min_pin_len; + ulong_t ei_total_public_memory; + ulong_t ei_free_public_memory; + ulong_t ei_total_private_memory; + ulong_t ei_free_private_memory; + crypto_version_t ei_hardware_version; + crypto_version_t ei_firmware_version; + uchar_t ei_time[CRYPTO_EXT_SIZE_TIME]; + int ei_hash_max_input_len; + int ei_hmac_max_input_len; +} crypto_provider_ext_info_t; + +typedef uint_t crypto_session_id_t; + +typedef enum cmd_type { + COPY_FROM_DATA, + COPY_TO_DATA, + COMPARE_TO_DATA, + MD5_DIGEST_DATA, + SHA1_DIGEST_DATA, + SHA2_DIGEST_DATA, + GHASH_DATA +} cmd_type_t; + +#define CRYPTO_DO_UPDATE 0x01 +#define CRYPTO_DO_FINAL 0x02 +#define CRYPTO_DO_MD5 0x04 +#define CRYPTO_DO_SHA1 0x08 +#define CRYPTO_DO_SIGN 0x10 +#define CRYPTO_DO_VERIFY 0x20 +#define CRYPTO_DO_SHA2 0x40 + +#define PROVIDER_OWNS_KEY_SCHEDULE 0x00000001 + +/* + * Common cryptographic status and error codes. + */ +#define CRYPTO_SUCCESS 0x00000000 +#define CRYPTO_CANCEL 0x00000001 +#define CRYPTO_HOST_MEMORY 0x00000002 +#define CRYPTO_GENERAL_ERROR 0x00000003 +#define CRYPTO_FAILED 0x00000004 +#define CRYPTO_ARGUMENTS_BAD 0x00000005 +#define CRYPTO_ATTRIBUTE_READ_ONLY 0x00000006 +#define CRYPTO_ATTRIBUTE_SENSITIVE 0x00000007 +#define CRYPTO_ATTRIBUTE_TYPE_INVALID 0x00000008 +#define CRYPTO_ATTRIBUTE_VALUE_INVALID 0x00000009 +#define CRYPTO_CANCELED 0x0000000A +#define CRYPTO_DATA_INVALID 0x0000000B +#define CRYPTO_DATA_LEN_RANGE 0x0000000C +#define CRYPTO_DEVICE_ERROR 0x0000000D +#define CRYPTO_DEVICE_MEMORY 0x0000000E +#define CRYPTO_DEVICE_REMOVED 0x0000000F +#define CRYPTO_ENCRYPTED_DATA_INVALID 0x00000010 +#define CRYPTO_ENCRYPTED_DATA_LEN_RANGE 0x00000011 +#define CRYPTO_KEY_HANDLE_INVALID 0x00000012 +#define CRYPTO_KEY_SIZE_RANGE 0x00000013 +#define CRYPTO_KEY_TYPE_INCONSISTENT 0x00000014 +#define CRYPTO_KEY_NOT_NEEDED 0x00000015 +#define CRYPTO_KEY_CHANGED 0x00000016 +#define CRYPTO_KEY_NEEDED 0x00000017 +#define CRYPTO_KEY_INDIGESTIBLE 0x00000018 +#define CRYPTO_KEY_FUNCTION_NOT_PERMITTED 0x00000019 +#define CRYPTO_KEY_NOT_WRAPPABLE 0x0000001A +#define CRYPTO_KEY_UNEXTRACTABLE 0x0000001B +#define CRYPTO_MECHANISM_INVALID 0x0000001C +#define CRYPTO_MECHANISM_PARAM_INVALID 0x0000001D +#define CRYPTO_OBJECT_HANDLE_INVALID 0x0000001E +#define CRYPTO_OPERATION_IS_ACTIVE 0x0000001F +#define CRYPTO_OPERATION_NOT_INITIALIZED 0x00000020 +#define CRYPTO_PIN_INCORRECT 0x00000021 +#define CRYPTO_PIN_INVALID 0x00000022 +#define CRYPTO_PIN_LEN_RANGE 0x00000023 +#define CRYPTO_PIN_EXPIRED 0x00000024 +#define CRYPTO_PIN_LOCKED 0x00000025 +#define CRYPTO_SESSION_CLOSED 0x00000026 +#define CRYPTO_SESSION_COUNT 0x00000027 +#define CRYPTO_SESSION_HANDLE_INVALID 0x00000028 +#define CRYPTO_SESSION_READ_ONLY 0x00000029 +#define CRYPTO_SESSION_EXISTS 0x0000002A +#define CRYPTO_SESSION_READ_ONLY_EXISTS 0x0000002B +#define CRYPTO_SESSION_READ_WRITE_SO_EXISTS 0x0000002C +#define CRYPTO_SIGNATURE_INVALID 0x0000002D +#define CRYPTO_SIGNATURE_LEN_RANGE 0x0000002E +#define CRYPTO_TEMPLATE_INCOMPLETE 0x0000002F +#define CRYPTO_TEMPLATE_INCONSISTENT 0x00000030 +#define CRYPTO_UNWRAPPING_KEY_HANDLE_INVALID 0x00000031 +#define CRYPTO_UNWRAPPING_KEY_SIZE_RANGE 0x00000032 +#define CRYPTO_UNWRAPPING_KEY_TYPE_INCONSISTENT 0x00000033 +#define CRYPTO_USER_ALREADY_LOGGED_IN 0x00000034 +#define CRYPTO_USER_NOT_LOGGED_IN 0x00000035 +#define CRYPTO_USER_PIN_NOT_INITIALIZED 0x00000036 +#define CRYPTO_USER_TYPE_INVALID 0x00000037 +#define CRYPTO_USER_ANOTHER_ALREADY_LOGGED_IN 0x00000038 +#define CRYPTO_USER_TOO_MANY_TYPES 0x00000039 +#define CRYPTO_WRAPPED_KEY_INVALID 0x0000003A +#define CRYPTO_WRAPPED_KEY_LEN_RANGE 0x0000003B +#define CRYPTO_WRAPPING_KEY_HANDLE_INVALID 0x0000003C +#define CRYPTO_WRAPPING_KEY_SIZE_RANGE 0x0000003D +#define CRYPTO_WRAPPING_KEY_TYPE_INCONSISTENT 0x0000003E +#define CRYPTO_RANDOM_SEED_NOT_SUPPORTED 0x0000003F +#define CRYPTO_RANDOM_NO_RNG 0x00000040 +#define CRYPTO_DOMAIN_PARAMS_INVALID 0x00000041 +#define CRYPTO_BUFFER_TOO_SMALL 0x00000042 +#define CRYPTO_INFORMATION_SENSITIVE 0x00000043 +#define CRYPTO_NOT_SUPPORTED 0x00000044 + +#define CRYPTO_QUEUED 0x00000045 +#define CRYPTO_BUFFER_TOO_BIG 0x00000046 +#define CRYPTO_INVALID_CONTEXT 0x00000047 +#define CRYPTO_INVALID_MAC 0x00000048 +#define CRYPTO_MECH_NOT_SUPPORTED 0x00000049 +#define CRYPTO_INCONSISTENT_ATTRIBUTE 0x0000004A +#define CRYPTO_NO_PERMISSION 0x0000004B +#define CRYPTO_INVALID_PROVIDER_ID 0x0000004C +#define CRYPTO_VERSION_MISMATCH 0x0000004D +#define CRYPTO_BUSY 0x0000004E +#define CRYPTO_UNKNOWN_PROVIDER 0x0000004F +#define CRYPTO_MODVERIFICATION_FAILED 0x00000050 +#define CRYPTO_OLD_CTX_TEMPLATE 0x00000051 +#define CRYPTO_WEAK_KEY 0x00000052 +#define CRYPTO_FIPS140_ERROR 0x00000053 +/* + * Don't forget to update CRYPTO_LAST_ERROR and the error_number_table[] + * in kernelUtil.c when new error code is added. + */ +#define CRYPTO_LAST_ERROR 0x00000053 + +/* + * Special values that can be used to indicate that information is unavailable + * or that there is not practical limit. These values can be used + * by fields of the SPI crypto_provider_ext_info(9S) structure. + * The value of CRYPTO_UNAVAILABLE_INFO should be the same as + * CK_UNAVAILABLE_INFO in the PKCS#11 spec. + */ +#define CRYPTO_UNAVAILABLE_INFO ((ulong_t)(-1)) +#define CRYPTO_EFFECTIVELY_INFINITE 0x0 + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRYPTO_COMMON_H */ diff --git a/include/sys/crypto/icp.h b/include/sys/crypto/icp.h new file mode 100644 index 000000000000..c7bb78e83623 --- /dev/null +++ b/include/sys/crypto/icp.h @@ -0,0 +1,41 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2016, Datto, Inc. All rights reserved. + */ + +#ifndef _SYS_CRYPTO_ALGS_H +#define _SYS_CRYPTO_ALGS_H + +int aes_mod_init(void); +int aes_mod_fini(void); + +int sha1_mod_init(void); +int sha1_mod_fini(void); + +int sha2_mod_init(void); +int sha2_mod_fini(void); + +int icp_init(void); +void icp_fini(void); + +#endif /* _SYS_CRYPTO_ALGS_H */ diff --git a/include/sys/dmu.h b/include/sys/dmu.h index 64d28334eac4..87a8a40eb782 100644 --- a/include/sys/dmu.h +++ b/include/sys/dmu.h @@ -334,10 +334,19 @@ typedef struct dmu_buf { */ uint64_t dmu_object_alloc(objset_t *os, dmu_object_type_t ot, int blocksize, dmu_object_type_t bonus_type, int bonus_len, dmu_tx_t *tx); +uint64_t dmu_object_alloc_dnsize(objset_t *os, dmu_object_type_t ot, + int blocksize, dmu_object_type_t bonus_type, int bonus_len, + int dnodesize, dmu_tx_t *tx); int dmu_object_claim(objset_t *os, uint64_t object, dmu_object_type_t ot, int blocksize, dmu_object_type_t bonus_type, int bonus_len, dmu_tx_t *tx); +int dmu_object_claim_dnsize(objset_t *os, uint64_t object, dmu_object_type_t ot, + int blocksize, dmu_object_type_t bonus_type, int bonus_len, + int dnodesize, dmu_tx_t *tx); int dmu_object_reclaim(objset_t *os, uint64_t object, dmu_object_type_t ot, int blocksize, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *txp); +int dmu_object_reclaim_dnsize(objset_t *os, uint64_t object, + dmu_object_type_t ot, int blocksize, dmu_object_type_t bonustype, + int bonuslen, int dnodesize, dmu_tx_t *txp); /* * Free an object from this objset. @@ -756,6 +765,7 @@ typedef struct dmu_object_info { uint8_t doi_compress; uint8_t doi_nblkptr; uint8_t doi_pad[4]; + uint64_t doi_dnodesize; uint64_t doi_physical_blocks_512; /* data + metadata, 512b blks */ uint64_t doi_max_offset; uint64_t doi_fill_count; /* number of non-empty blocks */ @@ -797,6 +807,8 @@ void dmu_object_info_from_db(dmu_buf_t *db, dmu_object_info_t *doi); void dmu_object_size_from_db(dmu_buf_t *db, uint32_t *blksize, u_longlong_t *nblk512); +void dmu_object_dnsize_from_db(dmu_buf_t *db, int *dnsize); + typedef struct dmu_objset_stats { uint64_t dds_num_clones; /* number of clones of this */ uint64_t dds_creation_txg; @@ -804,7 +816,7 @@ typedef struct dmu_objset_stats { dmu_objset_type_t dds_type; uint8_t dds_is_snapshot; uint8_t dds_inconsistent; - char dds_origin[MAXNAMELEN]; + char dds_origin[ZFS_MAX_DATASET_NAME_LEN]; } dmu_objset_stats_t; /* @@ -854,6 +866,7 @@ extern struct dsl_dataset *dmu_objset_ds(objset_t *os); extern void dmu_objset_name(objset_t *os, char *buf); extern dmu_objset_type_t dmu_objset_type(objset_t *os); extern uint64_t dmu_objset_id(objset_t *os); +extern uint64_t dmu_objset_dnodesize(objset_t *os); extern zfs_sync_type_t dmu_objset_syncprop(objset_t *os); extern zfs_logbias_op_t dmu_objset_logbias(objset_t *os); extern int dmu_snapshot_list_next(objset_t *os, int namelen, char *name, diff --git a/include/sys/dmu_impl.h b/include/sys/dmu_impl.h index 75d094f0812e..3e4423c8d6a0 100644 --- a/include/sys/dmu_impl.h +++ b/include/sys/dmu_impl.h @@ -24,7 +24,7 @@ */ /* * Copyright (c) 2012, Joyent, Inc. All rights reserved. - * Copyright (c) 2013 by Delphix. All rights reserved. + * Copyright (c) 2013, 2015 by Delphix. All rights reserved. */ #ifndef _SYS_DMU_IMPL_H @@ -268,10 +268,11 @@ typedef struct dmu_sendarg { uint64_t dsa_toguid; int dsa_err; dmu_pendop_t dsa_pending_op; - boolean_t dsa_incremental; uint64_t dsa_featureflags; uint64_t dsa_last_data_object; uint64_t dsa_last_data_offset; + uint64_t dsa_resume_object; + uint64_t dsa_resume_offset; } dmu_sendarg_t; void dmu_object_zapify(objset_t *, uint64_t, dmu_object_type_t, dmu_tx_t *); diff --git a/include/sys/dmu_objset.h b/include/sys/dmu_objset.h index 837a0d5107b7..81bb89bfc240 100644 --- a/include/sys/dmu_objset.h +++ b/include/sys/dmu_objset.h @@ -88,6 +88,7 @@ struct objset { list_node_t os_evicting_node; /* can change, under dsl_dir's locks: */ + uint64_t os_dnodesize; /* default dnode size for new objects */ enum zio_checksum os_checksum; enum zio_compress os_compress; uint8_t os_copies; @@ -106,6 +107,8 @@ struct objset { zil_header_t os_zil_header; list_t os_synced_dnodes; uint64_t os_flags; + uint64_t os_freed_dnodes; + boolean_t os_rescan_dnodes; /* Protected by os_obj_lock */ kmutex_t os_obj_lock; diff --git a/include/sys/dmu_send.h b/include/sys/dmu_send.h index 2442a1f8aab1..871f5625460e 100644 --- a/include/sys/dmu_send.h +++ b/include/sys/dmu_send.h @@ -36,10 +36,13 @@ struct vnode; struct dsl_dataset; struct drr_begin; struct avl_tree; +struct dmu_replay_record; -int dmu_send(const char *tosnap, const char *fromsnap, - boolean_t embedok, boolean_t large_block_ok, - int outfd, struct vnode *vp, offset_t *off); +extern const char *recv_clone_name; + +int dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok, + boolean_t large_block_ok, int outfd, uint64_t resumeobj, uint64_t resumeoff, + struct vnode *vp, offset_t *off); int dmu_send_estimate(struct dsl_dataset *ds, struct dsl_dataset *fromds, uint64_t *sizep); int dmu_send_estimate_from_txg(struct dsl_dataset *ds, uint64_t fromtxg, @@ -50,12 +53,14 @@ int dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap, typedef struct dmu_recv_cookie { struct dsl_dataset *drc_ds; + struct dmu_replay_record *drc_drr_begin; struct drr_begin *drc_drrb; const char *drc_tofs; const char *drc_tosnap; boolean_t drc_newfs; boolean_t drc_byteswap; boolean_t drc_force; + boolean_t drc_resumable; struct avl_tree *drc_guid_to_ds_map; zio_cksum_t drc_cksum; uint64_t drc_newsnapobj; @@ -63,8 +68,9 @@ typedef struct dmu_recv_cookie { cred_t *drc_cred; } dmu_recv_cookie_t; -int dmu_recv_begin(char *tofs, char *tosnap, struct drr_begin *drrb, - boolean_t force, char *origin, dmu_recv_cookie_t *drc); +int dmu_recv_begin(char *tofs, char *tosnap, + struct dmu_replay_record *drr_begin, + boolean_t force, boolean_t resumable, char *origin, dmu_recv_cookie_t *drc); int dmu_recv_stream(dmu_recv_cookie_t *drc, struct vnode *vp, offset_t *voffp, int cleanup_fd, uint64_t *action_handlep); int dmu_recv_end(dmu_recv_cookie_t *drc, void *owner); diff --git a/include/sys/dmu_traverse.h b/include/sys/dmu_traverse.h index 544b721e4612..c010edd440d9 100644 --- a/include/sys/dmu_traverse.h +++ b/include/sys/dmu_traverse.h @@ -54,6 +54,8 @@ typedef int (blkptr_cb_t)(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, int traverse_dataset(struct dsl_dataset *ds, uint64_t txg_start, int flags, blkptr_cb_t func, void *arg); +int traverse_dataset_resume(struct dsl_dataset *ds, uint64_t txg_start, + zbookmark_phys_t *resume, int flags, blkptr_cb_t func, void *arg); int traverse_dataset_destroyed(spa_t *spa, blkptr_t *blkptr, uint64_t txg_start, zbookmark_phys_t *resume, int flags, blkptr_cb_t func, void *arg); diff --git a/include/sys/dnode.h b/include/sys/dnode.h index c5250d51abf1..cee4ea783810 100644 --- a/include/sys/dnode.h +++ b/include/sys/dnode.h @@ -79,11 +79,18 @@ extern "C" { /* * Derived constants. */ -#define DNODE_SIZE (1 << DNODE_SHIFT) -#define DN_MAX_NBLKPTR ((DNODE_SIZE - DNODE_CORE_SIZE) >> SPA_BLKPTRSHIFT) -#define DN_MAX_BONUSLEN (DNODE_SIZE - DNODE_CORE_SIZE - (1 << SPA_BLKPTRSHIFT)) +#define DNODE_MIN_SIZE (1 << DNODE_SHIFT) +#define DNODE_MAX_SIZE (1 << DNODE_BLOCK_SHIFT) +#define DNODE_BLOCK_SIZE (1 << DNODE_BLOCK_SHIFT) +#define DNODE_MIN_SLOTS (DNODE_MIN_SIZE >> DNODE_SHIFT) +#define DNODE_MAX_SLOTS (DNODE_MAX_SIZE >> DNODE_SHIFT) +#define DN_BONUS_SIZE(dnsize) ((dnsize) - DNODE_CORE_SIZE - \ + (1 << SPA_BLKPTRSHIFT)) +#define DN_SLOTS_TO_BONUSLEN(slots) DN_BONUS_SIZE((slots) << DNODE_SHIFT) +#define DN_OLD_MAX_BONUSLEN (DN_BONUS_SIZE(DNODE_MIN_SIZE)) +#define DN_MAX_NBLKPTR ((DNODE_MIN_SIZE - DNODE_CORE_SIZE) >> SPA_BLKPTRSHIFT) #define DN_MAX_OBJECT (1ULL << DN_MAX_OBJECT_SHIFT) -#define DN_ZERO_BONUSLEN (DN_MAX_BONUSLEN + 1) +#define DN_ZERO_BONUSLEN (DN_BONUS_SIZE(DNODE_MAX_SIZE) + 1) #define DN_KILL_SPILLBLK (1) #define DNODES_PER_BLOCK_SHIFT (DNODE_BLOCK_SHIFT - DNODE_SHIFT) @@ -131,7 +138,8 @@ typedef struct dnode_phys { uint8_t dn_flags; /* DNODE_FLAG_* */ uint16_t dn_datablkszsec; /* data block size in 512b sectors */ uint16_t dn_bonuslen; /* length of dn_bonus */ - uint8_t dn_pad2[4]; + uint8_t dn_extra_slots; /* # of subsequent slots consumed */ + uint8_t dn_pad2[3]; /* accounting is protected by dn_dirty_mtx */ uint64_t dn_maxblkid; /* largest allocated block ID */ @@ -140,8 +148,11 @@ typedef struct dnode_phys { uint64_t dn_pad3[4]; /* - * The tail region is 448 bytes, and there are three ways to - * look at it. + * The tail region is 448 bytes for a 512 byte dnode, and + * correspondingly larger for larger dnode sizes. The spill + * block pointer, when present, is always at the end of the tail + * region. There are three ways this space may be used, using + * a 512 byte dnode for this diagram: * * 0 64 128 192 256 320 384 448 (offset) * +---------------+---------------+---------------+-------+ @@ -149,23 +160,27 @@ typedef struct dnode_phys { * +---------------+---------------+---------------+-------+ * | dn_blkptr[0] | dn_bonus[0..319] | * +---------------+-----------------------+---------------+ - * | dn_blkptr[0] | / | dn_spill | + * | dn_blkptr[0] | dn_bonus[0..191] | dn_spill | * +---------------+-----------------------+---------------+ */ union { - blkptr_t dn_blkptr[1+DN_MAX_BONUSLEN/sizeof (blkptr_t)]; + blkptr_t dn_blkptr[1+DN_OLD_MAX_BONUSLEN/sizeof (blkptr_t)]; struct { blkptr_t __dn_ignore1; - uint8_t dn_bonus[DN_MAX_BONUSLEN]; + uint8_t dn_bonus[DN_OLD_MAX_BONUSLEN]; }; struct { blkptr_t __dn_ignore2; - uint8_t __dn_ignore3[DN_MAX_BONUSLEN-sizeof (blkptr_t)]; + uint8_t __dn_ignore3[DN_OLD_MAX_BONUSLEN - + sizeof (blkptr_t)]; blkptr_t dn_spill; }; }; } dnode_phys_t; +#define DN_SPILL_BLKPTR(dnp) (blkptr_t *)((char *)(dnp) + \ + (((dnp)->dn_extra_slots + 1) << DNODE_SHIFT) - (1 << SPA_BLKPTRSHIFT)) + typedef struct dnode { /* * Protects the structure of the dnode, including the number of levels @@ -202,6 +217,7 @@ typedef struct dnode { uint32_t dn_datablksz; /* in bytes */ uint64_t dn_maxblkid; uint8_t dn_next_type[TXG_SIZE]; + uint8_t dn_num_slots; /* metadnode slots consumed on disk */ uint8_t dn_next_nblkptr[TXG_SIZE]; uint8_t dn_next_nlevels[TXG_SIZE]; uint8_t dn_next_indblkshift[TXG_SIZE]; @@ -299,7 +315,7 @@ void dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx); int dnode_hold(struct objset *dd, uint64_t object, void *ref, dnode_t **dnp); -int dnode_hold_impl(struct objset *dd, uint64_t object, int flag, +int dnode_hold_impl(struct objset *dd, uint64_t object, int flag, int dn_slots, void *ref, dnode_t **dnp); boolean_t dnode_add_ref(dnode_t *dn, void *ref); void dnode_rele(dnode_t *dn, void *ref); @@ -307,9 +323,9 @@ void dnode_rele_and_unlock(dnode_t *dn, void *tag); void dnode_setdirty(dnode_t *dn, dmu_tx_t *tx); void dnode_sync(dnode_t *dn, dmu_tx_t *tx); void dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, - dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx); + dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx); void dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, - dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx); + dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx); void dnode_free(dnode_t *dn, dmu_tx_t *tx); void dnode_byteswap(dnode_phys_t *dnp); void dnode_buf_byteswap(void *buf, size_t size); diff --git a/include/sys/dsl_dataset.h b/include/sys/dsl_dataset.h index a596642e3130..4d2f5e418bee 100644 --- a/include/sys/dsl_dataset.h +++ b/include/sys/dsl_dataset.h @@ -91,6 +91,25 @@ struct dsl_pool; */ #define DS_FIELD_LARGE_BLOCKS "org.open-zfs:large_blocks" +/* + * This field is present (with value=0) if this dataset may contain large + * dnodes (>512B). If it is present, then this dataset is counted in the + * refcount of the SPA_FEATURE_LARGE_DNODE feature. + */ +#define DS_FIELD_LARGE_DNODE "org.zfsonlinux:large_dnode" + +/* + * These fields are set on datasets that are in the middle of a resumable + * receive, and allow the sender to resume the send if it is interrupted. + */ +#define DS_FIELD_RESUME_FROMGUID "com.delphix:resume_fromguid" +#define DS_FIELD_RESUME_TONAME "com.delphix:resume_toname" +#define DS_FIELD_RESUME_TOGUID "com.delphix:resume_toguid" +#define DS_FIELD_RESUME_OBJECT "com.delphix:resume_object" +#define DS_FIELD_RESUME_OFFSET "com.delphix:resume_offset" +#define DS_FIELD_RESUME_BYTES "com.delphix:resume_bytes" +#define DS_FIELD_RESUME_EMBEDOK "com.delphix:resume_embedok" + /* * DS_FLAG_CI_DATASET is set if the dataset contains a file system whose * name lookups should be performed case-insensitively. @@ -184,6 +203,14 @@ typedef struct dsl_dataset { kmutex_t ds_sendstream_lock; list_t ds_sendstreams; + /* + * When in the middle of a resumable receive, tracks how much + * progress we have made. + */ + uint64_t ds_resume_object[TXG_SIZE]; + uint64_t ds_resume_offset[TXG_SIZE]; + uint64_t ds_resume_bytes[TXG_SIZE]; + /* Protected by our dsl_dir's dd_lock */ list_t ds_prop_cbs; @@ -200,7 +227,7 @@ typedef struct dsl_dataset { uint8_t ds_feature_activation_needed[SPA_FEATURES]; /* Protected by ds_lock; keep at end of struct for better locality */ - char ds_snapname[MAXNAMELEN]; + char ds_snapname[ZFS_MAX_DATASET_NAME_LEN]; } dsl_dataset_t; static inline dsl_dataset_phys_t * @@ -235,6 +262,8 @@ int dsl_dataset_own_obj(struct dsl_pool *dp, uint64_t dsobj, void dsl_dataset_disown(dsl_dataset_t *ds, void *tag); void dsl_dataset_name(dsl_dataset_t *ds, char *name); boolean_t dsl_dataset_tryown(dsl_dataset_t *ds, void *tag); +int dsl_dataset_namelen(dsl_dataset_t *ds); +boolean_t dsl_dataset_has_owner(dsl_dataset_t *ds); uint64_t dsl_dataset_create_sync(dsl_dir_t *pds, const char *lastname, dsl_dataset_t *origin, uint64_t flags, cred_t *, dmu_tx_t *); uint64_t dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin, @@ -315,6 +344,8 @@ int dsl_dataset_snap_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx, void dsl_dataset_set_refreservation_sync_impl(dsl_dataset_t *ds, zprop_source_t source, uint64_t value, dmu_tx_t *tx); void dsl_dataset_zapify(dsl_dataset_t *ds, dmu_tx_t *tx); +boolean_t dsl_dataset_is_zapified(dsl_dataset_t *ds); +boolean_t dsl_dataset_has_resume_receive_state(dsl_dataset_t *ds); int dsl_dataset_rollback(const char *fsname, void *owner, nvlist_t *result); void dsl_dataset_deactivate_feature(uint64_t dsobj, @@ -323,10 +354,10 @@ void dsl_dataset_deactivate_feature(uint64_t dsobj, #ifdef ZFS_DEBUG #define dprintf_ds(ds, fmt, ...) do { \ if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ - char *__ds_name = kmem_alloc(MAXNAMELEN, KM_SLEEP); \ + char *__ds_name = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); \ dsl_dataset_name(ds, __ds_name); \ dprintf("ds=%s " fmt, __ds_name, __VA_ARGS__); \ - kmem_free(__ds_name, MAXNAMELEN); \ + kmem_free(__ds_name, ZFS_MAX_DATASET_NAME_LEN); \ } \ _NOTE(CONSTCOND) } while (0) #else diff --git a/include/sys/dsl_dir.h b/include/sys/dsl_dir.h index a025c50a0737..fb299684c424 100644 --- a/include/sys/dsl_dir.h +++ b/include/sys/dsl_dir.h @@ -112,7 +112,7 @@ struct dsl_dir { int64_t dd_space_towrite[TXG_SIZE]; /* protected by dd_lock; keep at end of struct for better locality */ - char dd_myname[MAXNAMELEN]; + char dd_myname[ZFS_MAX_DATASET_NAME_LEN]; }; static inline dsl_dir_phys_t * @@ -176,11 +176,10 @@ boolean_t dsl_dir_is_zapified(dsl_dir_t *dd); #ifdef ZFS_DEBUG #define dprintf_dd(dd, fmt, ...) do { \ if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ - char *__ds_name = kmem_alloc(MAXNAMELEN + strlen(MOS_DIR_NAME) + 1, \ - KM_SLEEP); \ + char *__ds_name = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); \ dsl_dir_name(dd, __ds_name); \ dprintf("dd=%s " fmt, __ds_name, __VA_ARGS__); \ - kmem_free(__ds_name, MAXNAMELEN + strlen(MOS_DIR_NAME) + 1); \ + kmem_free(__ds_name, ZFS_MAX_DATASET_NAME_LEN); \ } \ _NOTE(CONSTCOND) } while (0) #else diff --git a/include/sys/fs/zfs.h b/include/sys/fs/zfs.h index 65dba125c941..b25d3016aea8 100644 --- a/include/sys/fs/zfs.h +++ b/include/sys/fs/zfs.h @@ -67,9 +67,13 @@ typedef enum dmu_objset_type { #define ZFS_TYPE_DATASET \ (ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME | ZFS_TYPE_SNAPSHOT) +/* + * All of these include the terminating NUL byte. + */ #define ZAP_MAXNAMELEN 256 #define ZAP_MAXVALUELEN (1024 * 8) #define ZAP_OLDMAXVALUELEN 1024 +#define ZFS_MAX_DATASET_NAME_LEN 256 /* * Dataset properties are identified by these constants and must be added to @@ -137,6 +141,7 @@ typedef enum { ZFS_PROP_DEDUP, ZFS_PROP_MLSLABEL, ZFS_PROP_SYNC, + ZFS_PROP_DNODESIZE, ZFS_PROP_REFRATIO, ZFS_PROP_WRITTEN, ZFS_PROP_CLONES, @@ -157,6 +162,7 @@ typedef enum { ZFS_PROP_REDUNDANT_METADATA, ZFS_PROP_OVERLAY, ZFS_PROP_PREV_SNAP, + ZFS_PROP_RECEIVE_RESUME_TOKEN, ZFS_NUM_PROPS } zfs_prop_t; @@ -204,6 +210,7 @@ typedef enum { ZPOOL_PROP_LEAKED, ZPOOL_PROP_MAXBLOCKSIZE, ZPOOL_PROP_TNAME, + ZPOOL_PROP_MAXDNODESIZE, ZPOOL_NUM_PROPS } zpool_prop_t; @@ -229,6 +236,7 @@ typedef enum { #define ZPROP_SOURCE_VAL_RECVD "$recvd" #define ZPROP_N_MORE_ERRORS "N_MORE_ERRORS" + /* * Dataset flag implemented as a special entry in the props zap object * indicating that the dataset has received properties on or after @@ -361,6 +369,16 @@ typedef enum { ZFS_XATTR_SA = 2 } zfs_xattr_type_t; +typedef enum { + ZFS_DNSIZE_LEGACY = 0, + ZFS_DNSIZE_AUTO = 1, + ZFS_DNSIZE_1K = 1024, + ZFS_DNSIZE_2K = 2048, + ZFS_DNSIZE_4K = 4096, + ZFS_DNSIZE_8K = 8192, + ZFS_DNSIZE_16K = 16384 +} zfs_dnsize_type_t; + typedef enum { ZFS_REDUNDANT_METADATA_ALL, ZFS_REDUNDANT_METADATA_MOST @@ -558,7 +576,17 @@ typedef struct zpool_rewind_policy { #define ZPOOL_CONFIG_VDEV_ASYNC_W_LAT_HISTO "vdev_async_w_lat_histo" #define ZPOOL_CONFIG_VDEV_SCRUB_LAT_HISTO "vdev_scrub_histo" - +/* Request size histograms */ +#define ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO "vdev_sync_ind_r_histo" +#define ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO "vdev_sync_ind_w_histo" +#define ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO "vdev_async_ind_r_histo" +#define ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO "vdev_async_ind_w_histo" +#define ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO "vdev_ind_scrub_histo" +#define ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO "vdev_sync_agg_r_histo" +#define ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO "vdev_sync_agg_w_histo" +#define ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO "vdev_async_agg_r_histo" +#define ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO "vdev_async_agg_w_histo" +#define ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO "vdev_agg_scrub_histo" #define ZPOOL_CONFIG_WHOLE_DISK "whole_disk" #define ZPOOL_CONFIG_ERRCOUNT "error_count" @@ -824,20 +852,33 @@ typedef struct vdev_stat_ex { * 2^37 nanoseconds = 134s. Timeouts will probably start kicking in * before this. */ -#define VDEV_HISTO_BUCKETS 37 +#define VDEV_L_HISTO_BUCKETS 37 /* Latency histo buckets */ +#define VDEV_RQ_HISTO_BUCKETS 25 /* Request size histo buckets */ + /* Amount of time in ZIO queue (ns) */ uint64_t vsx_queue_histo[ZIO_PRIORITY_NUM_QUEUEABLE] - [VDEV_HISTO_BUCKETS]; + [VDEV_L_HISTO_BUCKETS]; /* Total ZIO latency (ns). Includes queuing and disk access time */ - uint64_t vsx_total_histo[ZIO_TYPES][VDEV_HISTO_BUCKETS]; + uint64_t vsx_total_histo[ZIO_TYPES][VDEV_L_HISTO_BUCKETS]; /* Amount of time to read/write the disk (ns) */ - uint64_t vsx_disk_histo[ZIO_TYPES][VDEV_HISTO_BUCKETS]; + uint64_t vsx_disk_histo[ZIO_TYPES][VDEV_L_HISTO_BUCKETS]; + + /* "lookup the bucket for a value" histogram macros */ +#define HISTO(val, buckets) (val != 0 ? MIN(highbit64(val) - 1, \ + buckets - 1) : 0) +#define L_HISTO(a) HISTO(a, VDEV_L_HISTO_BUCKETS) +#define RQ_HISTO(a) HISTO(a, VDEV_RQ_HISTO_BUCKETS) + + /* Physical IO histogram */ + uint64_t vsx_ind_histo[ZIO_PRIORITY_NUM_QUEUEABLE] + [VDEV_RQ_HISTO_BUCKETS]; - /* "lookup the bucket for a value" macro */ -#define HISTO(a) (a != 0 ? MIN(highbit64(a) - 1, VDEV_HISTO_BUCKETS - 1) : 0) + /* Delegated (aggregated) physical IO histogram */ + uint64_t vsx_agg_histo[ZIO_PRIORITY_NUM_QUEUEABLE] + [VDEV_RQ_HISTO_BUCKETS]; } vdev_stat_ex_t; @@ -887,7 +928,7 @@ typedef struct ddt_histogram { */ typedef enum zfs_ioc { /* - * Illumos - 70/128 numbers reserved. + * Illumos - 71/128 numbers reserved. */ ZFS_IOC_FIRST = ('Z' << 8), ZFS_IOC = ZFS_IOC_FIRST, @@ -961,6 +1002,7 @@ typedef enum zfs_ioc { ZFS_IOC_BOOKMARK, ZFS_IOC_GET_BOOKMARKS, ZFS_IOC_DESTROY_BOOKMARKS, + ZFS_IOC_RECV_NEW, /* * Linux - 3/64 numbers reserved. @@ -981,7 +1023,7 @@ typedef enum zfs_ioc { /* * zvol ioctl to get dataset name */ -#define BLKZNAME _IOR(0x12, 125, char[ZFS_MAXNAMELEN]) +#define BLKZNAME _IOR(0x12, 125, char[ZFS_MAX_DATASET_NAME_LEN]) /* * Internal SPA load state. Used by FMA diagnosis engine. diff --git a/include/sys/policy.h b/include/sys/policy.h new file mode 100644 index 000000000000..23d7d4db77f2 --- /dev/null +++ b/include/sys/policy.h @@ -0,0 +1,60 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright 2015, Joyent, Inc. All rights reserved. + * Copyright (c) 2016, Lawrence Livermore National Security, LLC. + */ + +#ifndef _SYS_POLICY_H +#define _SYS_POLICY_H + +#ifdef _KERNEL + +#include +#include +#include +#include + +int secpolicy_nfs(const cred_t *); +int secpolicy_sys_config(const cred_t *, boolean_t); +int secpolicy_vnode_access2(const cred_t *, struct inode *, + uid_t, mode_t, mode_t); +int secpolicy_vnode_any_access(const cred_t *, struct inode *, uid_t); +int secpolicy_vnode_chown(const cred_t *, uid_t); +int secpolicy_vnode_create_gid(const cred_t *); +int secpolicy_vnode_remove(const cred_t *); +int secpolicy_vnode_setdac(const cred_t *, uid_t); +int secpolicy_vnode_setid_retain(const cred_t *, boolean_t); +int secpolicy_vnode_setids_setgids(const cred_t *, gid_t); +int secpolicy_zinject(const cred_t *); +int secpolicy_zfs(const cred_t *); +void secpolicy_setid_clear(vattr_t *, cred_t *); +int secpolicy_setid_setsticky_clear(struct inode *, vattr_t *, + const vattr_t *, cred_t *); +int secpolicy_xvattr(xvattr_t *, uid_t, cred_t *, vtype_t); +int secpolicy_vnode_setattr(cred_t *, struct inode *, struct vattr *, + const struct vattr *, int, int (void *, int, cred_t *), void *); +int secpolicy_basic_link(const cred_t *); + +#endif /* _KERNEL */ +#endif /* _SYS_POLICY_H */ diff --git a/include/sys/sa_impl.h b/include/sys/sa_impl.h index 6f2f1db6dcf9..b68b7610b25e 100644 --- a/include/sys/sa_impl.h +++ b/include/sys/sa_impl.h @@ -235,7 +235,7 @@ struct sa_handle { #define SA_BONUSTYPE_FROM_DB(db) \ (dmu_get_bonustype((dmu_buf_t *)db)) -#define SA_BLKPTR_SPACE (DN_MAX_BONUSLEN - sizeof (blkptr_t)) +#define SA_BLKPTR_SPACE (DN_OLD_MAX_BONUSLEN - sizeof (blkptr_t)) #define SA_LAYOUT_NUM(x, type) \ ((!IS_SA_BONUSTYPE(type) ? 0 : (((IS_SA_BONUSTYPE(type)) && \ diff --git a/include/sys/spa.h b/include/sys/spa.h index c80e8337ea3a..17bf76de83bf 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -35,6 +35,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -142,12 +143,6 @@ typedef struct dva { uint64_t dva_word[2]; } dva_t; -/* - * Each block has a 256-bit checksum -- strong enough for cryptographic hashes. - */ -typedef struct zio_cksum { - uint64_t zc_word[4]; -} zio_cksum_t; /* * Each block is described by its DVAs, time of birth, checksum, etc. @@ -440,35 +435,9 @@ _NOTE(CONSTCOND) } while (0) DVA_EQUAL(&(bp1)->blk_dva[1], &(bp2)->blk_dva[1]) && \ DVA_EQUAL(&(bp1)->blk_dva[2], &(bp2)->blk_dva[2])) -#define ZIO_CHECKSUM_EQUAL(zc1, zc2) \ - (0 == (((zc1).zc_word[0] - (zc2).zc_word[0]) | \ - ((zc1).zc_word[1] - (zc2).zc_word[1]) | \ - ((zc1).zc_word[2] - (zc2).zc_word[2]) | \ - ((zc1).zc_word[3] - (zc2).zc_word[3]))) - -#define ZIO_CHECKSUM_IS_ZERO(zc) \ - (0 == ((zc)->zc_word[0] | (zc)->zc_word[1] | \ - (zc)->zc_word[2] | (zc)->zc_word[3])) - -#define ZIO_CHECKSUM_BSWAP(zcp) \ -{ \ - (zcp)->zc_word[0] = BSWAP_64((zcp)->zc_word[0]); \ - (zcp)->zc_word[1] = BSWAP_64((zcp)->zc_word[1]); \ - (zcp)->zc_word[2] = BSWAP_64((zcp)->zc_word[2]); \ - (zcp)->zc_word[3] = BSWAP_64((zcp)->zc_word[3]); \ -} - #define DVA_IS_VALID(dva) (DVA_GET_ASIZE(dva) != 0) -#define ZIO_SET_CHECKSUM(zcp, w0, w1, w2, w3) \ -{ \ - (zcp)->zc_word[0] = w0; \ - (zcp)->zc_word[1] = w1; \ - (zcp)->zc_word[2] = w2; \ - (zcp)->zc_word[3] = w3; \ -} - #define BP_IDENTITY(bp) (ASSERT(!BP_IS_EMBEDDED(bp)), &(bp)->blk_dva[0]) #define BP_IS_GANG(bp) \ (BP_IS_EMBEDDED(bp) ? B_FALSE : DVA_GET_GANG(BP_IDENTITY(bp))) @@ -853,6 +822,7 @@ extern boolean_t spa_is_root(spa_t *spa); extern boolean_t spa_writeable(spa_t *spa); extern boolean_t spa_has_pending_synctask(spa_t *spa); extern int spa_maxblocksize(spa_t *spa); +extern int spa_maxdnodesize(spa_t *spa); extern void zfs_blkptr_verify(spa_t *spa, const blkptr_t *bp); extern int spa_mode(spa_t *spa); diff --git a/include/sys/spa_checksum.h b/include/sys/spa_checksum.h new file mode 100644 index 000000000000..b87990105a71 --- /dev/null +++ b/include/sys/spa_checksum.h @@ -0,0 +1,72 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SPA_CHECKSUM_H +#define _SPA_CHECKSUM_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Each block has a 256-bit checksum -- strong enough for cryptographic hashes. + */ +typedef struct zio_cksum { + uint64_t zc_word[4]; +} zio_cksum_t; + +#define ZIO_SET_CHECKSUM(zcp, w0, w1, w2, w3) \ +{ \ + (zcp)->zc_word[0] = w0; \ + (zcp)->zc_word[1] = w1; \ + (zcp)->zc_word[2] = w2; \ + (zcp)->zc_word[3] = w3; \ +} + +#define ZIO_CHECKSUM_EQUAL(zc1, zc2) \ + (0 == (((zc1).zc_word[0] - (zc2).zc_word[0]) | \ + ((zc1).zc_word[1] - (zc2).zc_word[1]) | \ + ((zc1).zc_word[2] - (zc2).zc_word[2]) | \ + ((zc1).zc_word[3] - (zc2).zc_word[3]))) + +#define ZIO_CHECKSUM_IS_ZERO(zc) \ + (0 == ((zc)->zc_word[0] | (zc)->zc_word[1] | \ + (zc)->zc_word[2] | (zc)->zc_word[3])) + +#define ZIO_CHECKSUM_BSWAP(zcp) \ +{ \ + (zcp)->zc_word[0] = BSWAP_64((zcp)->zc_word[0]); \ + (zcp)->zc_word[1] = BSWAP_64((zcp)->zc_word[1]); \ + (zcp)->zc_word[2] = BSWAP_64((zcp)->zc_word[2]); \ + (zcp)->zc_word[3] = BSWAP_64((zcp)->zc_word[3]); \ +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/sys/spa_impl.h b/include/sys/spa_impl.h index 2eb5fa12a85c..59cb44de215d 100644 --- a/include/sys/spa_impl.h +++ b/include/sys/spa_impl.h @@ -126,7 +126,7 @@ struct spa { /* * Fields protected by spa_namespace_lock. */ - char spa_name[MAXNAMELEN]; /* pool name */ + char spa_name[ZFS_MAX_DATASET_NAME_LEN]; /* pool name */ char *spa_comment; /* comment */ avl_node_t spa_avl; /* node in spa_namespace_avl */ nvlist_t *spa_config; /* last synced config */ diff --git a/include/sys/trace_acl.h b/include/sys/trace_acl.h index f1d8eb78c0bf..bd19c5a1669d 100644 --- a/include/sys/trace_acl.h +++ b/include/sys/trace_acl.h @@ -31,6 +31,7 @@ #define _TRACE_ACL_H #include +#include #include /* @@ -55,10 +56,7 @@ DECLARE_EVENT_CLASS(zfs_ace_class, __field(uint_t, z_seq) __field(uint64_t, z_mapcnt) __field(uint64_t, z_size) - __field(uint64_t, z_links) __field(uint64_t, z_pflags) - __field(uint64_t, z_uid) - __field(uint64_t, z_gid) __field(uint32_t, z_sync_cnt) __field(mode_t, z_mode) __field(boolean_t, z_is_sa) @@ -66,6 +64,8 @@ DECLARE_EVENT_CLASS(zfs_ace_class, __field(boolean_t, z_is_ctldir) __field(boolean_t, z_is_stale) + __field(uint32_t, i_uid) + __field(uint32_t, i_gid) __field(unsigned long, i_ino) __field(unsigned int, i_nlink) __field(u64, i_version) @@ -91,10 +91,7 @@ DECLARE_EVENT_CLASS(zfs_ace_class, __entry->z_seq = zn->z_seq; __entry->z_mapcnt = zn->z_mapcnt; __entry->z_size = zn->z_size; - __entry->z_links = zn->z_links; __entry->z_pflags = zn->z_pflags; - __entry->z_uid = zn->z_uid; - __entry->z_gid = zn->z_gid; __entry->z_sync_cnt = zn->z_sync_cnt; __entry->z_mode = zn->z_mode; __entry->z_is_sa = zn->z_is_sa; @@ -102,6 +99,8 @@ DECLARE_EVENT_CLASS(zfs_ace_class, __entry->z_is_ctldir = zn->z_is_ctldir; __entry->z_is_stale = zn->z_is_stale; + __entry->i_uid = zfs_uid_read(ZTOI(zn)); + __entry->i_gid = zfs_gid_read(ZTOI(zn)); __entry->i_ino = zn->z_inode.i_ino; __entry->i_nlink = zn->z_inode.i_nlink; __entry->i_version = zn->z_inode.i_version; @@ -119,23 +118,23 @@ DECLARE_EVENT_CLASS(zfs_ace_class, ), TP_printk("zn { id %llu unlinked %u atime_dirty %u " "zn_prefetch %u moved %u blksz %u seq %u " - "mapcnt %llu size %llu links %llu pflags %llu " - "uid %llu gid %llu sync_cnt %u mode 0x%x is_sa %d " + "mapcnt %llu size %llu pflags %llu " + "sync_cnt %u mode 0x%x is_sa %d " "is_mapped %d is_ctldir %d is_stale %d inode { " - "ino %lu nlink %u version %llu size %lli blkbits %u " - "bytes %u mode 0x%x generation %x } } ace { type %u " - "flags %u access_mask %u } mask_matched %u", + "uid %u gid %u ino %lu nlink %u version %llu size %lli " + "blkbits %u bytes %u mode 0x%x generation %x } } " + "ace { type %u flags %u access_mask %u } mask_matched %u", __entry->z_id, __entry->z_unlinked, __entry->z_atime_dirty, __entry->z_zn_prefetch, __entry->z_moved, __entry->z_blksz, __entry->z_seq, __entry->z_mapcnt, __entry->z_size, - __entry->z_links, __entry->z_pflags, __entry->z_uid, - __entry->z_gid, __entry->z_sync_cnt, __entry->z_mode, + __entry->z_pflags, __entry->z_sync_cnt, __entry->z_mode, __entry->z_is_sa, __entry->z_is_mapped, - __entry->z_is_ctldir, __entry->z_is_stale, __entry->i_ino, - __entry->i_nlink, __entry->i_version, __entry->i_size, - __entry->i_blkbits, __entry->i_bytes, __entry->i_mode, - __entry->i_generation, __entry->z_type, __entry->z_flags, - __entry->z_access_mask, __entry->mask_matched) + __entry->z_is_ctldir, __entry->z_is_stale, __entry->i_uid, + __entry->i_gid, __entry->i_ino, __entry->i_nlink, + __entry->i_version, __entry->i_size, __entry->i_blkbits, + __entry->i_bytes, __entry->i_mode, __entry->i_generation, + __entry->z_type, __entry->z_flags, __entry->z_access_mask, + __entry->mask_matched) ); #define DEFINE_ACE_EVENT(name) \ diff --git a/include/sys/trace_arc.h b/include/sys/trace_arc.h index 55dbdf19b055..0fca639e1493 100644 --- a/include/sys/trace_arc.h +++ b/include/sys/trace_arc.h @@ -34,6 +34,7 @@ #include #include +#include /* For ZIO macros */ /* * Generic support for one argument tracepoints of the form: @@ -115,86 +116,6 @@ DEFINE_ARC_BUF_HDR_EVENT(zfs_l2arc__miss); * zio_t *, ...); */ -#define ZIO_TP_STRUCT_ENTRY \ - __field(zio_type_t, zio_type) \ - __field(int, zio_cmd) \ - __field(zio_priority_t, zio_priority) \ - __field(uint64_t, zio_size) \ - __field(uint64_t, zio_orig_size) \ - __field(uint64_t, zio_offset) \ - __field(hrtime_t, zio_timestamp) \ - __field(hrtime_t, zio_delta) \ - __field(uint64_t, zio_delay) \ - __field(enum zio_flag, zio_flags) \ - __field(enum zio_stage, zio_stage) \ - __field(enum zio_stage, zio_pipeline) \ - __field(enum zio_flag, zio_orig_flags) \ - __field(enum zio_stage, zio_orig_stage) \ - __field(enum zio_stage, zio_orig_pipeline) \ - __field(uint8_t, zio_reexecute) \ - __field(uint64_t, zio_txg) \ - __field(int, zio_error) \ - __field(uint64_t, zio_ena) \ - \ - __field(enum zio_checksum, zp_checksum) \ - __field(enum zio_compress, zp_compress) \ - __field(dmu_object_type_t, zp_type) \ - __field(uint8_t, zp_level) \ - __field(uint8_t, zp_copies) \ - __field(boolean_t, zp_dedup) \ - __field(boolean_t, zp_dedup_verify) \ - __field(boolean_t, zp_nopwrite) - -#define ZIO_TP_FAST_ASSIGN \ - __entry->zio_type = zio->io_type; \ - __entry->zio_cmd = zio->io_cmd; \ - __entry->zio_priority = zio->io_priority; \ - __entry->zio_size = zio->io_size; \ - __entry->zio_orig_size = zio->io_orig_size; \ - __entry->zio_offset = zio->io_offset; \ - __entry->zio_timestamp = zio->io_timestamp; \ - __entry->zio_delta = zio->io_delta; \ - __entry->zio_delay = zio->io_delay; \ - __entry->zio_flags = zio->io_flags; \ - __entry->zio_stage = zio->io_stage; \ - __entry->zio_pipeline = zio->io_pipeline; \ - __entry->zio_orig_flags = zio->io_orig_flags; \ - __entry->zio_orig_stage = zio->io_orig_stage; \ - __entry->zio_orig_pipeline = zio->io_orig_pipeline; \ - __entry->zio_reexecute = zio->io_reexecute; \ - __entry->zio_txg = zio->io_txg; \ - __entry->zio_error = zio->io_error; \ - __entry->zio_ena = zio->io_ena; \ - \ - __entry->zp_checksum = zio->io_prop.zp_checksum; \ - __entry->zp_compress = zio->io_prop.zp_compress; \ - __entry->zp_type = zio->io_prop.zp_type; \ - __entry->zp_level = zio->io_prop.zp_level; \ - __entry->zp_copies = zio->io_prop.zp_copies; \ - __entry->zp_dedup = zio->io_prop.zp_dedup; \ - __entry->zp_nopwrite = zio->io_prop.zp_nopwrite; \ - __entry->zp_dedup_verify = zio->io_prop.zp_dedup_verify; - -#define ZIO_TP_PRINTK_FMT \ - "zio { type %u cmd %i prio %u size %llu orig_size %llu " \ - "offset %llu timestamp %llu delta %llu delay %llu " \ - "flags 0x%x stage 0x%x pipeline 0x%x orig_flags 0x%x " \ - "orig_stage 0x%x orig_pipeline 0x%x reexecute %u " \ - "txg %llu error %d ena %llu prop { checksum %u compress %u " \ - "type %u level %u copies %u dedup %u dedup_verify %u nopwrite %u } }" - -#define ZIO_TP_PRINTK_ARGS \ - __entry->zio_type, __entry->zio_cmd, __entry->zio_priority, \ - __entry->zio_size, __entry->zio_orig_size, __entry->zio_offset, \ - __entry->zio_timestamp, __entry->zio_delta, __entry->zio_delay, \ - __entry->zio_flags, __entry->zio_stage, __entry->zio_pipeline, \ - __entry->zio_orig_flags, __entry->zio_orig_stage, \ - __entry->zio_orig_pipeline, __entry->zio_reexecute, \ - __entry->zio_txg, __entry->zio_error, __entry->zio_ena, \ - __entry->zp_checksum, __entry->zp_compress, __entry->zp_type, \ - __entry->zp_level, __entry->zp_copies, __entry->zp_dedup, \ - __entry->zp_dedup_verify, __entry->zp_nopwrite - DECLARE_EVENT_CLASS(zfs_l2arc_rw_class, TP_PROTO(vdev_t *vd, zio_t *zio), TP_ARGS(vd, zio), diff --git a/include/sys/trace_common.h b/include/sys/trace_common.h new file mode 100644 index 000000000000..6922d1a1810a --- /dev/null +++ b/include/sys/trace_common.h @@ -0,0 +1,112 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * This file contains commonly used trace macros. Feel free to add and use + * them in your tracepoint headers. + */ + +#ifndef _SYS_TRACE_COMMON_H +#define _SYS_TRACE_COMMON_H +#include + +/* ZIO macros */ +#define ZIO_TP_STRUCT_ENTRY \ + __field(zio_type_t, zio_type) \ + __field(int, zio_cmd) \ + __field(zio_priority_t, zio_priority) \ + __field(uint64_t, zio_size) \ + __field(uint64_t, zio_orig_size) \ + __field(uint64_t, zio_offset) \ + __field(hrtime_t, zio_timestamp) \ + __field(hrtime_t, zio_delta) \ + __field(uint64_t, zio_delay) \ + __field(enum zio_flag, zio_flags) \ + __field(enum zio_stage, zio_stage) \ + __field(enum zio_stage, zio_pipeline) \ + __field(enum zio_flag, zio_orig_flags) \ + __field(enum zio_stage, zio_orig_stage) \ + __field(enum zio_stage, zio_orig_pipeline) \ + __field(uint8_t, zio_reexecute) \ + __field(uint64_t, zio_txg) \ + __field(int, zio_error) \ + __field(uint64_t, zio_ena) \ + \ + __field(enum zio_checksum, zp_checksum) \ + __field(enum zio_compress, zp_compress) \ + __field(dmu_object_type_t, zp_type) \ + __field(uint8_t, zp_level) \ + __field(uint8_t, zp_copies) \ + __field(boolean_t, zp_dedup) \ + __field(boolean_t, zp_dedup_verify) \ + __field(boolean_t, zp_nopwrite) + +#define ZIO_TP_FAST_ASSIGN \ + __entry->zio_type = zio->io_type; \ + __entry->zio_cmd = zio->io_cmd; \ + __entry->zio_priority = zio->io_priority; \ + __entry->zio_size = zio->io_size; \ + __entry->zio_orig_size = zio->io_orig_size; \ + __entry->zio_offset = zio->io_offset; \ + __entry->zio_timestamp = zio->io_timestamp; \ + __entry->zio_delta = zio->io_delta; \ + __entry->zio_delay = zio->io_delay; \ + __entry->zio_flags = zio->io_flags; \ + __entry->zio_stage = zio->io_stage; \ + __entry->zio_pipeline = zio->io_pipeline; \ + __entry->zio_orig_flags = zio->io_orig_flags; \ + __entry->zio_orig_stage = zio->io_orig_stage; \ + __entry->zio_orig_pipeline = zio->io_orig_pipeline; \ + __entry->zio_reexecute = zio->io_reexecute; \ + __entry->zio_txg = zio->io_txg; \ + __entry->zio_error = zio->io_error; \ + __entry->zio_ena = zio->io_ena; \ + \ + __entry->zp_checksum = zio->io_prop.zp_checksum; \ + __entry->zp_compress = zio->io_prop.zp_compress; \ + __entry->zp_type = zio->io_prop.zp_type; \ + __entry->zp_level = zio->io_prop.zp_level; \ + __entry->zp_copies = zio->io_prop.zp_copies; \ + __entry->zp_dedup = zio->io_prop.zp_dedup; \ + __entry->zp_nopwrite = zio->io_prop.zp_nopwrite; \ + __entry->zp_dedup_verify = zio->io_prop.zp_dedup_verify; + +#define ZIO_TP_PRINTK_FMT \ + "zio { type %u cmd %i prio %u size %llu orig_size %llu " \ + "offset %llu timestamp %llu delta %llu delay %llu " \ + "flags 0x%x stage 0x%x pipeline 0x%x orig_flags 0x%x " \ + "orig_stage 0x%x orig_pipeline 0x%x reexecute %u " \ + "txg %llu error %d ena %llu prop { checksum %u compress %u " \ + "type %u level %u copies %u dedup %u dedup_verify %u nopwrite %u } }" + +#define ZIO_TP_PRINTK_ARGS \ + __entry->zio_type, __entry->zio_cmd, __entry->zio_priority, \ + __entry->zio_size, __entry->zio_orig_size, __entry->zio_offset, \ + __entry->zio_timestamp, __entry->zio_delta, __entry->zio_delay, \ + __entry->zio_flags, __entry->zio_stage, __entry->zio_pipeline, \ + __entry->zio_orig_flags, __entry->zio_orig_stage, \ + __entry->zio_orig_pipeline, __entry->zio_reexecute, \ + __entry->zio_txg, __entry->zio_error, __entry->zio_ena, \ + __entry->zp_checksum, __entry->zp_compress, __entry->zp_type, \ + __entry->zp_level, __entry->zp_copies, __entry->zp_dedup, \ + __entry->zp_dedup_verify, __entry->zp_nopwrite + +#endif /* _SYS_TRACE_COMMON_H */ diff --git a/include/sys/trace_zio.h b/include/sys/trace_zio.h new file mode 100644 index 000000000000..6db6cefe27ef --- /dev/null +++ b/include/sys/trace_zio.h @@ -0,0 +1,87 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +#include + +#if defined(_KERNEL) && defined(HAVE_DECLARE_EVENT_CLASS) + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM zfs + +#undef TRACE_SYSTEM_VAR +#define TRACE_SYSTEM_VAR zfs_zio + +#if !defined(_TRACE_ZIO_H) || defined(TRACE_HEADER_MULTI_READ) +#define _TRACE_ZIO_H + +#include +#include +#include /* For ZIO macros */ + +TRACE_EVENT(zfs_zio__delay__miss, + TP_PROTO(zio_t *zio, hrtime_t now), + TP_ARGS(zio, now), + TP_STRUCT__entry( + ZIO_TP_STRUCT_ENTRY + __field(hrtime_t, now) + ), + TP_fast_assign( + ZIO_TP_FAST_ASSIGN + __entry->now = now; + ), + TP_printk("now %llu " ZIO_TP_PRINTK_FMT, __entry->now, + ZIO_TP_PRINTK_ARGS) +); + +TRACE_EVENT(zfs_zio__delay__hit, + TP_PROTO(zio_t *zio, hrtime_t now, hrtime_t diff), + TP_ARGS(zio, now, diff), + TP_STRUCT__entry( + ZIO_TP_STRUCT_ENTRY + __field(hrtime_t, now) + __field(hrtime_t, diff) + ), + TP_fast_assign( + ZIO_TP_FAST_ASSIGN + __entry->now = now; + __entry->diff = diff; + ), + TP_printk("now %llu diff %llu " ZIO_TP_PRINTK_FMT, __entry->now, + __entry->diff, ZIO_TP_PRINTK_ARGS) +); + +TRACE_EVENT(zfs_zio__delay__skip, + TP_PROTO(zio_t *zio), + TP_ARGS(zio), + TP_STRUCT__entry(ZIO_TP_STRUCT_ENTRY), + TP_fast_assign(ZIO_TP_FAST_ASSIGN), + TP_printk(ZIO_TP_PRINTK_FMT, ZIO_TP_PRINTK_ARGS) +); + +#endif /* _TRACE_ZIO_H */ + +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH sys +#define TRACE_INCLUDE_FILE trace_zio +#include + +#endif /* _KERNEL && HAVE_DECLARE_EVENT_CLASS */ diff --git a/include/sys/vdev_disk.h b/include/sys/vdev_disk.h index d5a1889d2a26..c6d5f92b22bc 100644 --- a/include/sys/vdev_disk.h +++ b/include/sys/vdev_disk.h @@ -37,9 +37,11 @@ typedef struct vdev_disk { struct block_device *vd_bdev; } vdev_disk_t; +#ifndef __linux__ extern int vdev_disk_physio(struct block_device *, caddr_t, size_t, uint64_t, int); extern int vdev_disk_read_rootlabel(char *, char *, nvlist_t **); +#endif #endif /* _KERNEL */ #endif /* _SYS_VDEV_DISK_H */ diff --git a/include/sys/vdev_raidz.h b/include/sys/vdev_raidz.h new file mode 100644 index 000000000000..06e21af41309 --- /dev/null +++ b/include/sys/vdev_raidz.h @@ -0,0 +1,64 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2016 Gvozden Neskovic . + */ + +#ifndef _SYS_VDEV_RAIDZ_H +#define _SYS_VDEV_RAIDZ_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct zio; +struct raidz_map; +#if !defined(_KERNEL) +struct kernel_param {}; +#endif + +/* + * vdev_raidz interface + */ +struct raidz_map * vdev_raidz_map_alloc(struct zio *, uint64_t, uint64_t, + uint64_t); +void vdev_raidz_map_free(struct raidz_map *); +void vdev_raidz_generate_parity(struct raidz_map *); +int vdev_raidz_reconstruct(struct raidz_map *, const int *, int); + +/* + * vdev_raidz_math interface + */ +void vdev_raidz_math_init(void); +void vdev_raidz_math_fini(void); +struct raidz_impl_ops * vdev_raidz_math_get_ops(void); +int vdev_raidz_math_generate(struct raidz_map *); +int vdev_raidz_math_reconstruct(struct raidz_map *, + const int *, const int *, const int); +int vdev_raidz_impl_set(const char *); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_VDEV_RAIDZ_H */ diff --git a/include/sys/vdev_raidz_impl.h b/include/sys/vdev_raidz_impl.h new file mode 100644 index 000000000000..af09aa643897 --- /dev/null +++ b/include/sys/vdev_raidz_impl.h @@ -0,0 +1,359 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. + */ + +#ifndef _VDEV_RAIDZ_H +#define _VDEV_RAIDZ_H + +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define CODE_P (0U) +#define CODE_Q (1U) +#define CODE_R (2U) + +#define PARITY_P (1U) +#define PARITY_PQ (2U) +#define PARITY_PQR (3U) + +#define TARGET_X (0U) +#define TARGET_Y (1U) +#define TARGET_Z (2U) + +/* + * Parity generation methods indexes + */ +enum raidz_math_gen_op { + RAIDZ_GEN_P = 0, + RAIDZ_GEN_PQ, + RAIDZ_GEN_PQR, + RAIDZ_GEN_NUM = 3 +}; +/* + * Data reconstruction methods indexes + */ +enum raidz_rec_op { + RAIDZ_REC_P = 0, + RAIDZ_REC_Q, + RAIDZ_REC_R, + RAIDZ_REC_PQ, + RAIDZ_REC_PR, + RAIDZ_REC_QR, + RAIDZ_REC_PQR, + RAIDZ_REC_NUM = 7 +}; + +extern const char *raidz_gen_name[RAIDZ_GEN_NUM]; +extern const char *raidz_rec_name[RAIDZ_REC_NUM]; + +/* + * Methods used to define raidz implementation + * + * @raidz_gen_f Parity generation function + * @par1 pointer to raidz_map + * @raidz_rec_f Data reconstruction function + * @par1 pointer to raidz_map + * @par2 array of reconstruction targets + * @will_work_f Function returns TRUE if impl. is supported on the system + * @init_impl_f Function is called once on init + * @fini_impl_f Function is called once on fini + */ +typedef void (*raidz_gen_f)(void *); +typedef int (*raidz_rec_f)(void *, const int *); +typedef boolean_t (*will_work_f)(void); +typedef void (*init_impl_f)(void); +typedef void (*fini_impl_f)(void); + +#define RAIDZ_IMPL_NAME_MAX (16) + +typedef struct raidz_impl_ops { + init_impl_f init; + fini_impl_f fini; + raidz_gen_f gen[RAIDZ_GEN_NUM]; /* Parity generate functions */ + raidz_rec_f rec[RAIDZ_REC_NUM]; /* Data reconstruction functions */ + will_work_f is_supported; /* Support check function */ + char name[RAIDZ_IMPL_NAME_MAX]; /* Name of the implementation */ +} raidz_impl_ops_t; + +typedef struct raidz_col { + size_t rc_devidx; /* child device index for I/O */ + size_t rc_offset; /* device offset */ + size_t rc_size; /* I/O size */ + void *rc_data; /* I/O data */ + void *rc_gdata; /* used to store the "good" version */ + int rc_error; /* I/O error for this device */ + unsigned int rc_tried; /* Did we attempt this I/O column? */ + unsigned int rc_skipped; /* Did we skip this I/O column? */ +} raidz_col_t; + +typedef struct raidz_map { + size_t rm_cols; /* Regular column count */ + size_t rm_scols; /* Count including skipped columns */ + size_t rm_bigcols; /* Number of oversized columns */ + size_t rm_asize; /* Actual total I/O size */ + size_t rm_missingdata; /* Count of missing data devices */ + size_t rm_missingparity; /* Count of missing parity devices */ + size_t rm_firstdatacol; /* First data column/parity count */ + size_t rm_nskip; /* Skipped sectors for padding */ + size_t rm_skipstart; /* Column index of padding start */ + void *rm_datacopy; /* rm_asize-buffer of copied data */ + size_t rm_reports; /* # of referencing checksum reports */ + unsigned int rm_freed; /* map no longer has referencing ZIO */ + unsigned int rm_ecksuminjected; /* checksum error was injected */ + raidz_impl_ops_t *rm_ops; /* RAIDZ math operations */ + raidz_col_t rm_col[1]; /* Flexible array of I/O columns */ +} raidz_map_t; + +#define RAIDZ_ORIGINAL_IMPL (INT_MAX) + +extern const raidz_impl_ops_t vdev_raidz_scalar_impl; +#if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */ +extern const raidz_impl_ops_t vdev_raidz_sse2_impl; +#endif +#if defined(__x86_64) && defined(HAVE_SSSE3) /* only x86_64 for now */ +extern const raidz_impl_ops_t vdev_raidz_ssse3_impl; +#endif +#if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */ +extern const raidz_impl_ops_t vdev_raidz_avx2_impl; +#endif + +/* + * Commonly used raidz_map helpers + * + * raidz_parity Returns parity of the RAIDZ block + * raidz_ncols Returns number of columns the block spans + * raidz_nbigcols Returns number of big columns columns + * raidz_col_p Returns pointer to a column + * raidz_col_size Returns size of a column + * raidz_big_size Returns size of big columns + * raidz_short_size Returns size of short columns + */ +#define raidz_parity(rm) ((rm)->rm_firstdatacol) +#define raidz_ncols(rm) ((rm)->rm_cols) +#define raidz_nbigcols(rm) ((rm)->rm_bigcols) +#define raidz_col_p(rm, c) ((rm)->rm_col + (c)) +#define raidz_col_size(rm, c) ((rm)->rm_col[c].rc_size) +#define raidz_big_size(rm) (raidz_col_size(rm, CODE_P)) +#define raidz_short_size(rm) (raidz_col_size(rm, raidz_ncols(rm)-1)) + +/* + * Macro defines an RAIDZ parity generation method + * + * @code parity the function produce + * @impl name of the implementation + */ +#define _RAIDZ_GEN_WRAP(code, impl) \ +static void \ +impl ## _gen_ ## code(void *rmp) \ +{ \ + raidz_map_t *rm = (raidz_map_t *) rmp; \ + raidz_generate_## code ## _impl(rm); \ +} + +/* + * Macro defines an RAIDZ data reconstruction method + * + * @code parity the function produce + * @impl name of the implementation + */ +#define _RAIDZ_REC_WRAP(code, impl) \ +static int \ +impl ## _rec_ ## code(void *rmp, const int *tgtidx) \ +{ \ + raidz_map_t *rm = (raidz_map_t *) rmp; \ + return (raidz_reconstruct_## code ## _impl(rm, tgtidx)); \ +} + +/* + * Define all gen methods for an implementation + * + * @impl name of the implementation + */ +#define DEFINE_GEN_METHODS(impl) \ + _RAIDZ_GEN_WRAP(p, impl); \ + _RAIDZ_GEN_WRAP(pq, impl); \ + _RAIDZ_GEN_WRAP(pqr, impl) + +/* + * Define all rec functions for an implementation + * + * @impl name of the implementation + */ +#define DEFINE_REC_METHODS(impl) \ + _RAIDZ_REC_WRAP(p, impl); \ + _RAIDZ_REC_WRAP(q, impl); \ + _RAIDZ_REC_WRAP(r, impl); \ + _RAIDZ_REC_WRAP(pq, impl); \ + _RAIDZ_REC_WRAP(pr, impl); \ + _RAIDZ_REC_WRAP(qr, impl); \ + _RAIDZ_REC_WRAP(pqr, impl) + +#define RAIDZ_GEN_METHODS(impl) \ +{ \ + [RAIDZ_GEN_P] = & impl ## _gen_p, \ + [RAIDZ_GEN_PQ] = & impl ## _gen_pq, \ + [RAIDZ_GEN_PQR] = & impl ## _gen_pqr \ +} + +#define RAIDZ_REC_METHODS(impl) \ +{ \ + [RAIDZ_REC_P] = & impl ## _rec_p, \ + [RAIDZ_REC_Q] = & impl ## _rec_q, \ + [RAIDZ_REC_R] = & impl ## _rec_r, \ + [RAIDZ_REC_PQ] = & impl ## _rec_pq, \ + [RAIDZ_REC_PR] = & impl ## _rec_pr, \ + [RAIDZ_REC_QR] = & impl ## _rec_qr, \ + [RAIDZ_REC_PQR] = & impl ## _rec_pqr \ +} + + +typedef struct raidz_impl_kstat { + uint64_t gen[RAIDZ_GEN_NUM]; /* gen method speed B/s */ + uint64_t rec[RAIDZ_REC_NUM]; /* rec method speed B/s */ +} raidz_impl_kstat_t; + +/* + * Enumerate various multiplication constants + * used in reconstruction methods + */ +typedef enum raidz_mul_info { + /* Reconstruct Q */ + MUL_Q_X = 0, + /* Reconstruct R */ + MUL_R_X = 0, + /* Reconstruct PQ */ + MUL_PQ_X = 0, + MUL_PQ_Y = 1, + /* Reconstruct PR */ + MUL_PR_X = 0, + MUL_PR_Y = 1, + /* Reconstruct QR */ + MUL_QR_XQ = 0, + MUL_QR_X = 1, + MUL_QR_YQ = 2, + MUL_QR_Y = 3, + /* Reconstruct PQR */ + MUL_PQR_XP = 0, + MUL_PQR_XQ = 1, + MUL_PQR_XR = 2, + MUL_PQR_YU = 3, + MUL_PQR_YP = 4, + MUL_PQR_YQ = 5, + + MUL_CNT = 6 +} raidz_mul_info_t; + +/* + * Powers of 2 in the Galois field. + */ +extern const uint8_t vdev_raidz_pow2[256] __attribute__((aligned(256))); +/* Logs of 2 in the Galois field defined above. */ +extern const uint8_t vdev_raidz_log2[256] __attribute__((aligned(256))); + +/* + * Multiply a given number by 2 raised to the given power. + */ +static inline uint8_t +vdev_raidz_exp2(const uint8_t a, const unsigned exp) +{ + if (a == 0) + return (0); + + return (vdev_raidz_pow2[(exp + (unsigned) vdev_raidz_log2[a]) % 255]); +} + +/* + * Galois Field operations. + * + * gf_exp2 - computes 2 raised to the given power + * gf_exp2 - computes 4 raised to the given power + * gf_mul - multiplication + * gf_div - division + * gf_inv - multiplicative inverse + */ +typedef unsigned gf_t; +typedef unsigned gf_log_t; + +static inline gf_t +gf_mul(const gf_t a, const gf_t b) +{ + gf_log_t logsum; + + if (a == 0 || b == 0) + return (0); + + logsum = (gf_log_t) vdev_raidz_log2[a] + (gf_log_t) vdev_raidz_log2[b]; + + return ((gf_t) vdev_raidz_pow2[logsum % 255]); +} + +static inline gf_t +gf_div(const gf_t a, const gf_t b) +{ + gf_log_t logsum; + + ASSERT3U(b, >, 0); + if (a == 0) + return (0); + + logsum = (gf_log_t) 255 + (gf_log_t) vdev_raidz_log2[a] - + (gf_log_t) vdev_raidz_log2[b]; + + return ((gf_t) vdev_raidz_pow2[logsum % 255]); +} + +static inline gf_t +gf_inv(const gf_t a) +{ + gf_log_t logsum; + + ASSERT3U(a, >, 0); + + logsum = (gf_log_t) 255 - (gf_log_t) vdev_raidz_log2[a]; + + return ((gf_t) vdev_raidz_pow2[logsum]); +} + +static inline gf_t +gf_exp2(gf_log_t exp) +{ + return (vdev_raidz_pow2[exp % 255]); +} + +static inline gf_t +gf_exp4(gf_log_t exp) +{ + ASSERT3U(exp, <=, 255); + return ((gf_t) vdev_raidz_pow2[(2 * exp) % 255]); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _VDEV_RAIDZ_H */ diff --git a/include/sys/zap.h b/include/sys/zap.h index ed60b86dbd71..7009473979b2 100644 --- a/include/sys/zap.h +++ b/include/sys/zap.h @@ -129,16 +129,30 @@ typedef enum zap_flags { * MT_FIRST/MT_BEST matching will find entries that match without * regard to case (eg. looking for "foo" can find an entry "Foo"). * Eventually, other flags will permit unicode normalization as well. + * + * dnodesize specifies the on-disk size of the dnode for the new zapobj. + * Valid values are multiples of 512 up to DNODE_MAX_SIZE. */ uint64_t zap_create(objset_t *ds, dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx); +uint64_t zap_create_dnsize(objset_t *ds, dmu_object_type_t ot, + dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx); uint64_t zap_create_norm(objset_t *ds, int normflags, dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx); +uint64_t zap_create_norm_dnsize(objset_t *ds, int normflags, + dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, + int dnodesize, dmu_tx_t *tx); uint64_t zap_create_flags(objset_t *os, int normflags, zap_flags_t flags, dmu_object_type_t ot, int leaf_blockshift, int indirect_blockshift, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx); +uint64_t zap_create_flags_dnsize(objset_t *os, int normflags, + zap_flags_t flags, dmu_object_type_t ot, int leaf_blockshift, + int indirect_blockshift, dmu_object_type_t bonustype, int bonuslen, + int dnodesize, dmu_tx_t *tx); uint64_t zap_create_link(objset_t *os, dmu_object_type_t ot, uint64_t parent_obj, const char *name, dmu_tx_t *tx); +uint64_t zap_create_link_dnsize(objset_t *os, dmu_object_type_t ot, + uint64_t parent_obj, const char *name, int dnodesize, dmu_tx_t *tx); /* * Initialize an already-allocated object. @@ -152,9 +166,14 @@ void mzap_create_impl(objset_t *os, uint64_t obj, int normflags, */ int zap_create_claim(objset_t *ds, uint64_t obj, dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx); +int zap_create_claim_dnsize(objset_t *ds, uint64_t obj, dmu_object_type_t ot, + dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx); int zap_create_claim_norm(objset_t *ds, uint64_t obj, int normflags, dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx); +int zap_create_claim_norm_dnsize(objset_t *ds, uint64_t obj, + int normflags, dmu_object_type_t ot, + dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx); /* * The zapobj passed in must be a valid ZAP object for all of the @@ -344,7 +363,7 @@ typedef struct { boolean_t za_normalization_conflict; uint64_t za_num_integers; uint64_t za_first_integer; /* no sign extension for <8byte ints */ - char za_name[MAXNAMELEN]; + char za_name[ZAP_MAXNAMELEN]; } zap_attribute_t; /* diff --git a/include/sys/zfs_context.h b/include/sys/zfs_context.h index fc15d70421db..7224294c7813 100644 --- a/include/sys/zfs_context.h +++ b/include/sys/zfs_context.h @@ -62,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -445,7 +446,9 @@ typedef enum kmem_cbrc { /* * Task queues */ -typedef struct taskq taskq_t; + +#define TASKQ_NAMELEN 31 + typedef uintptr_t taskqid_t; typedef void (task_func_t)(void *); @@ -457,6 +460,25 @@ typedef struct taskq_ent { uintptr_t tqent_flags; } taskq_ent_t; +typedef struct taskq { + char tq_name[TASKQ_NAMELEN + 1]; + kmutex_t tq_lock; + krwlock_t tq_threadlock; + kcondvar_t tq_dispatch_cv; + kcondvar_t tq_wait_cv; + kthread_t **tq_threadlist; + int tq_flags; + int tq_active; + int tq_nthreads; + int tq_nalloc; + int tq_minalloc; + int tq_maxalloc; + kcondvar_t tq_maxalloc_cv; + int tq_maxalloc_wait; + taskq_ent_t *tq_freelist; + taskq_ent_t tq_task; +} taskq_t; + #define TQENT_FLAG_PREALLOC 0x1 /* taskq_dispatch_ent used */ #define TASKQ_PREPOPULATE 0x0001 @@ -651,6 +673,8 @@ extern uint64_t physmem; extern int highbit64(uint64_t i); extern int lowbit64(uint64_t i); +extern int highbit(ulong_t i); +extern int lowbit(ulong_t i); extern int random_get_bytes(uint8_t *ptr, size_t len); extern int random_get_pseudo_bytes(uint8_t *ptr, size_t len); @@ -658,6 +682,8 @@ extern void kernel_init(int); extern void kernel_fini(void); extern void thread_init(void); extern void thread_fini(void); +extern void random_init(void); +extern void random_fini(void); struct spa; extern void nicenum(uint64_t num, char *buf); @@ -738,6 +764,7 @@ extern int zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr); extern int zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr); extern int zfs_secpolicy_destroy_perms(const char *name, cred_t *cr); +extern int secpolicy_zfs(const cred_t *cr); extern zoneid_t getzoneid(void); /* SID stuff */ diff --git a/include/sys/zfs_ioctl.h b/include/sys/zfs_ioctl.h index 601a9a70c580..5157c6704f4b 100644 --- a/include/sys/zfs_ioctl.h +++ b/include/sys/zfs_ioctl.h @@ -20,7 +20,8 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. + * Copyright 2016 RackTop Systems. */ #ifndef _SYS_ZFS_IOCTL_H @@ -90,14 +91,16 @@ typedef enum drr_headertype { * Feature flags for zfs send streams (flags in drr_versioninfo) */ -#define DMU_BACKUP_FEATURE_DEDUP (1<<0) -#define DMU_BACKUP_FEATURE_DEDUPPROPS (1<<1) -#define DMU_BACKUP_FEATURE_SA_SPILL (1<<2) +#define DMU_BACKUP_FEATURE_DEDUP (1 << 0) +#define DMU_BACKUP_FEATURE_DEDUPPROPS (1 << 1) +#define DMU_BACKUP_FEATURE_SA_SPILL (1 << 2) /* flags #3 - #15 are reserved for incompatible closed-source implementations */ -#define DMU_BACKUP_FEATURE_EMBED_DATA (1<<16) -#define DMU_BACKUP_FEATURE_EMBED_DATA_LZ4 (1<<17) +#define DMU_BACKUP_FEATURE_EMBED_DATA (1 << 16) +#define DMU_BACKUP_FEATURE_EMBED_DATA_LZ4 (1 << 17) /* flag #18 is reserved for a Delphix feature */ -#define DMU_BACKUP_FEATURE_LARGE_BLOCKS (1<<19) +#define DMU_BACKUP_FEATURE_LARGE_BLOCKS (1 << 19) +#define DMU_BACKUP_FEATURE_RESUMING (1 << 20) +#define DMU_BACKUP_FEATURE_LARGE_DNODE (1 << 21) /* * Mask of all supported backup features @@ -105,11 +108,16 @@ typedef enum drr_headertype { #define DMU_BACKUP_FEATURE_MASK (DMU_BACKUP_FEATURE_DEDUP | \ DMU_BACKUP_FEATURE_DEDUPPROPS | DMU_BACKUP_FEATURE_SA_SPILL | \ DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_EMBED_DATA_LZ4 | \ - DMU_BACKUP_FEATURE_LARGE_BLOCKS) + DMU_BACKUP_FEATURE_RESUMING | DMU_BACKUP_FEATURE_LARGE_BLOCKS | \ + DMU_BACKUP_FEATURE_LARGE_DNODE) /* Are all features in the given flag word currently supported? */ #define DMU_STREAM_SUPPORTED(x) (!((x) & ~DMU_BACKUP_FEATURE_MASK)) +typedef enum dmu_send_resume_token_version { + ZFS_SEND_RESUME_TOKEN_VERSION = 1 +} dmu_send_resume_token_version_t; + /* * The drr_versioninfo field of the dmu_replay_record has the * following layout: @@ -129,8 +137,22 @@ typedef enum drr_headertype { #define DMU_BACKUP_MAGIC 0x2F5bacbacULL +/* + * Send stream flags. Bits 24-31 are reserved for vendor-specific + * implementations and should not be used. + */ #define DRR_FLAG_CLONE (1<<0) #define DRR_FLAG_CI_DATA (1<<1) +/* + * This send stream, if it is a full send, includes the FREE and FREEOBJECT + * records that are created by the sending process. This means that the send + * stream can be received as a clone, even though it is not an incremental. + * This is not implemented as a feature flag, because the receiving side does + * not need to have implemented it to receive this stream; it is fully backwards + * compatible. We need a flag, though, because full send streams without it + * cannot necessarily be received as a clone correctly. + */ +#define DRR_FLAG_FREERECORDS (1<<2) /* * flags in the drr_checksumflags field in the DRR_WRITE and @@ -173,7 +195,8 @@ typedef struct dmu_replay_record { uint32_t drr_bonuslen; uint8_t drr_checksumtype; uint8_t drr_compress; - uint8_t drr_pad[6]; + uint8_t drr_dn_slots; + uint8_t drr_pad[5]; uint64_t drr_toguid; /* bonus content follows */ } drr_object; @@ -287,6 +310,7 @@ typedef struct zinject_record { uint32_t zi_iotype; int32_t zi_duration; uint64_t zi_timer; + uint64_t zi_nlanes; uint32_t zi_cmd; uint32_t zi_pad; } zinject_record_t; @@ -331,6 +355,12 @@ typedef enum zfs_case { ZFS_CASE_MIXED } zfs_case_t; +/* + * Note: this struct must have the same layout in 32-bit and 64-bit, so + * that 32-bit processes (like /sbin/zfs) can pass it to the 64-bit + * kernel. Therefore, we add padding to it so that no "hidden" padding + * is automatically added on 64-bit (but not on 32-bit). + */ typedef struct zfs_cmd { char zc_name[MAXPATHLEN]; /* name of pool or dataset */ uint64_t zc_nvlist_src; /* really (char *) */ diff --git a/include/sys/zfs_znode.h b/include/sys/zfs_znode.h index 43b591915f2c..a12675d6f583 100644 --- a/include/sys/zfs_znode.h +++ b/include/sys/zfs_znode.h @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012 by Delphix. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. */ #ifndef _SYS_FS_ZFS_ZNODE_H @@ -138,17 +138,6 @@ extern "C" { #define ZFS_SHARES_DIR "SHARES" #define ZFS_SA_ATTRS "SA_ATTRS" -/* - * Path component length - * - * The generic fs code uses MAXNAMELEN to represent - * what the largest component length is. Unfortunately, - * this length includes the terminating NULL. ZFS needs - * to tell the users via pathconf() and statvfs() what the - * true maximum length of a component is, excluding the NULL. - */ -#define ZFS_MAXNAMELEN (MAXNAMELEN - 1) - /* * Convert mode bits (zp_mode) to BSD-style DT_* values for storing in * the directory entries. On Linux systems this value is already @@ -196,18 +185,15 @@ typedef struct znode { uint_t z_blksz; /* block size in bytes */ uint_t z_seq; /* modification sequence number */ uint64_t z_mapcnt; /* number of pages mapped to file */ + uint64_t z_dnodesize; /* dnode size */ uint64_t z_size; /* file size (cached) */ - uint64_t z_links; /* file links (cached) */ uint64_t z_pflags; /* pflags (cached) */ - uint64_t z_uid; /* uid fuid (cached) */ - uint64_t z_gid; /* gid fuid (cached) */ uint32_t z_sync_cnt; /* synchronous open count */ mode_t z_mode; /* mode (cached) */ kmutex_t z_acl_lock; /* acl data lock */ zfs_acl_t *z_acl_cached; /* cached acl */ krwlock_t z_xattr_lock; /* xattr data lock */ nvlist_t *z_xattr_cached; /* cached xattrs */ - struct znode *z_xattr_parent; /* xattr parent znode */ list_node_t z_link_node; /* all znodes in fs link */ sa_handle_t *z_sa_hdl; /* handle to sa data */ boolean_t z_is_sa; /* are we native sa? */ diff --git a/include/sys/zil.h b/include/sys/zil.h index 65b14f1cd6a2..1f04d1833664 100644 --- a/include/sys/zil.h +++ b/include/sys/zil.h @@ -172,6 +172,19 @@ typedef enum zil_create { (txtype) == TX_ACL || \ (txtype) == TX_WRITE2) +/* + * The number of dnode slots consumed by the object is stored in the 8 + * unused upper bits of the object ID. We subtract 1 from the value + * stored on disk for compatibility with implementations that don't + * support large dnodes. The slot count for a single-slot dnode will + * contain 0 for those bits to preserve the log record format for + * "small" dnodes. + */ +#define LR_FOID_GET_SLOTS(oid) (BF64_GET((oid), 56, 8) + 1) +#define LR_FOID_SET_SLOTS(oid, x) BF64_SET((oid), 56, 8, (x) - 1) +#define LR_FOID_GET_OBJ(oid) BF64_GET((oid), 0, DN_MAX_OBJECT_SHIFT) +#define LR_FOID_SET_OBJ(oid, x) BF64_SET((oid), 0, DN_MAX_OBJECT_SHIFT, (x)) + /* * Format of log records. * The fields are carefully defined to allow them to be aligned diff --git a/include/sys/zio.h b/include/sys/zio.h index 9790b4a9008c..d5a1494df2a5 100644 --- a/include/sys/zio.h +++ b/include/sys/zio.h @@ -22,7 +22,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. - * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. */ @@ -400,7 +400,8 @@ struct zio { zio_transform_t *io_transform_stack; /* Callback info */ - zio_done_func_t *io_ready; + zio_done_func_t *io_ready; + zio_done_func_t *io_children_ready; zio_done_func_t *io_physdone; zio_done_func_t *io_done; void *io_private; @@ -420,6 +421,7 @@ struct zio { uint64_t io_offset; hrtime_t io_timestamp; /* submitted at */ + hrtime_t io_target_timestamp; hrtime_t io_delta; /* vdev queue service delta */ hrtime_t io_delay; /* Device access time (disk or */ /* file). */ @@ -467,9 +469,10 @@ extern zio_t *zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, void *data, extern zio_t *zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data, uint64_t size, const zio_prop_t *zp, - zio_done_func_t *ready, zio_done_func_t *physdone, zio_done_func_t *done, - void *private, - zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb); + zio_done_func_t *ready, zio_done_func_t *children_ready, + zio_done_func_t *physdone, zio_done_func_t *done, + void *private, zio_priority_t priority, enum zio_flag flags, + const zbookmark_phys_t *zb); extern zio_t *zio_rewrite(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data, uint64_t size, zio_done_func_t *done, void *private, @@ -510,6 +513,8 @@ extern int zio_wait(zio_t *zio); extern void zio_nowait(zio_t *zio); extern void zio_execute(zio_t *zio); extern void zio_interrupt(zio_t *zio); +extern void zio_delay_init(zio_t *zio); +extern void zio_delay_interrupt(zio_t *zio); extern zio_t *zio_walk_parents(zio_t *cio); extern zio_t *zio_walk_children(zio_t *pio); @@ -572,7 +577,7 @@ extern int zio_handle_fault_injection(zio_t *zio, int error); extern int zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error); extern int zio_handle_label_injection(zio_t *zio, int error); extern void zio_handle_ignored_writes(zio_t *zio); -extern uint64_t zio_handle_io_delay(zio_t *zio); +extern hrtime_t zio_handle_io_delay(zio_t *zio); /* * Checksum ereport functions diff --git a/include/sys/zvol.h b/include/sys/zvol.h index c3e386f0b79e..00ed220d3b17 100644 --- a/include/sys/zvol.h +++ b/include/sys/zvol.h @@ -32,6 +32,8 @@ #define ZVOL_OBJ 1ULL #define ZVOL_ZAP_OBJ 2ULL +extern void *zvol_tag; + extern void zvol_create_minors(spa_t *spa, const char *name, boolean_t async); extern void zvol_remove_minors(spa_t *spa, const char *name, boolean_t async); extern void zvol_rename_minors(spa_t *spa, const char *oldname, diff --git a/include/zfeature_common.h b/include/zfeature_common.h index d481a28a8ffe..41cfdf8070c5 100644 --- a/include/zfeature_common.h +++ b/include/zfeature_common.h @@ -50,6 +50,7 @@ typedef enum spa_feature { SPA_FEATURE_BOOKMARKS, SPA_FEATURE_FS_SS_LIMIT, SPA_FEATURE_LARGE_BLOCKS, + SPA_FEATURE_LARGE_DNODE, SPA_FEATURES } spa_feature_t; diff --git a/include/zfs_fletcher.h b/include/zfs_fletcher.h index b49df0cf4f0f..afc3936c0a48 100644 --- a/include/zfs_fletcher.h +++ b/include/zfs_fletcher.h @@ -27,7 +27,7 @@ #define _ZFS_FLETCHER_H #include -#include +#include #ifdef __cplusplus extern "C" { @@ -45,6 +45,33 @@ void fletcher_4_incremental_native(const void *, uint64_t, zio_cksum_t *); void fletcher_4_incremental_byteswap(const void *, uint64_t, zio_cksum_t *); +int fletcher_4_impl_set(const char *selector); +void fletcher_4_init(void); +void fletcher_4_fini(void); + +/* + * fletcher checksum struct + */ +typedef struct fletcher_4_func { + void (*init)(zio_cksum_t *); + void (*fini)(zio_cksum_t *); + void (*compute)(const void *, uint64_t, zio_cksum_t *); + void (*compute_byteswap)(const void *, uint64_t, zio_cksum_t *); + boolean_t (*valid)(void); + const char *name; +} fletcher_4_ops_t; + +#if defined(HAVE_SSE2) +extern const fletcher_4_ops_t fletcher_4_sse2_ops; +#endif + +#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) +extern const fletcher_4_ops_t fletcher_4_ssse3_ops; +#endif + +#if defined(HAVE_AVX) && defined(HAVE_AVX2) +extern const fletcher_4_ops_t fletcher_4_avx2_ops; +#endif #ifdef __cplusplus } diff --git a/lib/Makefile.am b/lib/Makefile.am index 8e7caf2a16e9..3aec922c93ca 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -1,7 +1,7 @@ # NB: GNU Automake Manual, Chapter 8.3.5: Libtool Convenience Libraries -# These five libraries are intermediary build components. -SUBDIRS = libspl libavl libefi libshare libunicode +# These six libraries are intermediary build components. +SUBDIRS = libspl libavl libefi libshare libunicode libicp # These four libraries, which are installed as the final build product, -# incorporate the five convenience libraries given above. +# incorporate the six convenience libraries given above. SUBDIRS += libuutil libnvpair libzpool libzfs_core libzfs diff --git a/lib/libefi/rdwr_efi.c b/lib/libefi/rdwr_efi.c index 67de216c03c3..e9fb15ee2a6e 100644 --- a/lib/libefi/rdwr_efi.c +++ b/lib/libefi/rdwr_efi.c @@ -154,7 +154,8 @@ struct dk_map2 default_vtoc_map[NDKMAP] = { #if defined(_SUNOS_VTOC_16) #if defined(i386) || defined(__amd64) || defined(__arm) || \ - defined(__powerpc) || defined(__sparc) || defined(__s390__) + defined(__powerpc) || defined(__sparc) || defined(__s390__) || \ + defined(__mips__) { V_BOOT, V_UNMNT }, /* i - 8 */ { V_ALTSCTR, 0 }, /* j - 9 */ @@ -1105,7 +1106,7 @@ check_input(struct dk_gpt *vtoc) int efi_use_whole_disk(int fd) { - struct dk_gpt *efi_label; + struct dk_gpt *efi_label = NULL; int rval; int i; uint_t resv_index = 0, data_index = 0; @@ -1114,6 +1115,8 @@ efi_use_whole_disk(int fd) rval = efi_alloc_and_read(fd, &efi_label); if (rval < 0) { + if (efi_label != NULL) + efi_free(efi_label); return (rval); } diff --git a/lib/libicp/Makefile.am b/lib/libicp/Makefile.am new file mode 100644 index 000000000000..41457fd5276b --- /dev/null +++ b/lib/libicp/Makefile.am @@ -0,0 +1,78 @@ +include $(top_srcdir)/config/Rules.am + +VPATH = \ + $(top_srcdir)/module/icp \ + $(top_srcdir)/lib/libicp + +AM_CFLAGS += $(DEBUG_STACKFLAGS) $(FRAME_LARGER_THAN) + +DEFAULT_INCLUDES += \ + -I$(top_srcdir)/include \ + -I$(top_srcdir)/module/icp/include \ + -I$(top_srcdir)/lib/libspl/include + +noinst_LTLIBRARIES = libicp.la + +if TARGET_ASM_X86_64 +ASM_SOURCES_C = asm-x86_64/aes/aeskey.c +ASM_SOURCES_AS = \ + asm-x86_64/aes/aes_amd64.S \ + asm-x86_64/aes/aes_intel.S \ + asm-x86_64/modes/gcm_intel.S \ + asm-x86_64/sha1/sha1-x86_64.S \ + asm-x86_64/sha2/sha256_impl.S +endif + +if TARGET_ASM_I386 +ASM_SOURCES_C = +ASM_SOURCES_AS = +endif + +if TARGET_ASM_GENERIC +ASM_SOURCES_C = +ASM_SOURCES_AS = +endif + +USER_C = + +USER_ASM = + +KERNEL_C = \ + spi/kcf_spi.c \ + api/kcf_ctxops.c \ + api/kcf_digest.c \ + api/kcf_cipher.c \ + api/kcf_miscapi.c \ + api/kcf_mac.c \ + algs/aes/aes_impl.c \ + algs/aes/aes_modes.c \ + algs/modes/modes.c \ + algs/modes/cbc.c \ + algs/modes/gcm.c \ + algs/modes/ctr.c \ + algs/modes/ccm.c \ + algs/modes/ecb.c \ + algs/sha1/sha1.c \ + algs/sha2/sha2.c \ + illumos-crypto.c \ + io/aes.c \ + io/sha1_mod.c \ + io/sha2_mod.c \ + os/modhash.c \ + os/modconf.c \ + core/kcf_sched.c \ + core/kcf_prov_lib.c \ + core/kcf_callprov.c \ + core/kcf_mech_tabs.c \ + core/kcf_prov_tabs.c \ + $(ASM_SOURCES_C) + +KERNEL_ASM = $(ASM_SOURCES_AS) + +nodist_libicp_la_SOURCES = \ + $(USER_C) \ + $(USER_ASM) \ + $(KERNEL_C) \ + $(KERNEL_ASM) + +libicp_la_LIBADD = -lrt diff --git a/lib/libspl/include/Makefile.am b/lib/libspl/include/Makefile.am index 626a8f46b001..7882c1b9c6d3 100644 --- a/lib/libspl/include/Makefile.am +++ b/lib/libspl/include/Makefile.am @@ -12,7 +12,6 @@ libspl_HEADERS = \ $(top_srcdir)/lib/libspl/include/limits.h \ $(top_srcdir)/lib/libspl/include/locale.h \ $(top_srcdir)/lib/libspl/include/note.h \ - $(top_srcdir)/lib/libspl/include/priv.h \ $(top_srcdir)/lib/libspl/include/statcommon.h \ $(top_srcdir)/lib/libspl/include/stdio.h \ $(top_srcdir)/lib/libspl/include/stdlib.h \ diff --git a/lib/libspl/include/sys/byteorder.h b/lib/libspl/include/sys/byteorder.h index 528d2d208d6c..7ef1c42959f8 100644 --- a/lib/libspl/include/sys/byteorder.h +++ b/lib/libspl/include/sys/byteorder.h @@ -128,6 +128,28 @@ extern in_port_t ntohs(in_port_t); #define BE_64(x) BSWAP_64(x) #endif +#ifdef _BIG_ENDIAN +static __inline__ uint64_t +htonll(uint64_t n) { + return (n); +} + +static __inline__ uint64_t +ntohll(uint64_t n) { + return (n); +} +#else +static __inline__ uint64_t +htonll(uint64_t n) { + return ((((uint64_t)htonl(n)) << 32) + htonl(n >> 32)); +} + +static __inline__ uint64_t +ntohll(uint64_t n) { + return ((((uint64_t)ntohl(n)) << 32) + ntohl(n >> 32)); +} +#endif + /* * Macros to read unaligned values from a specific byte order to * native byte order diff --git a/lib/libspl/include/sys/file.h b/lib/libspl/include/sys/file.h index 163a4dca665a..b5d985bda264 100644 --- a/lib/libspl/include/sys/file.h +++ b/lib/libspl/include/sys/file.h @@ -33,7 +33,7 @@ #define FREAD 1 #define FWRITE 2 -// #define FAPPEND 8 +// #define FAPPEND 8 #define FCREAT O_CREAT #define FTRUNC O_TRUNC diff --git a/lib/libspl/include/sys/isa_defs.h b/lib/libspl/include/sys/isa_defs.h index 61f4cd96a93d..3bca5cf02e1b 100644 --- a/lib/libspl/include/sys/isa_defs.h +++ b/lib/libspl/include/sys/isa_defs.h @@ -172,7 +172,28 @@ extern "C" { #define _BIG_ENDIAN #define _SUNOS_VTOC_16 -#else /* Currently x86_64, i386, arm, powerpc, s390, and sparc are supported */ +/* MIPS arch specific defines */ +#elif defined(__mips__) + +#if defined(__MIPSEB__) +#define _BIG_ENDIAN +#elif defined(__MIPSEL__) +#define _LITTLE_ENDIAN +#else +#error MIPS no endian specified +#endif + +#ifndef _LP64 +#define _ILP32 +#endif + +#define _SUNOS_VTOC_16 + +#else +/* + * Currently supported: + * x86_64, i386, arm, powerpc, s390, sparc, and mips + */ #error "Unsupported ISA type" #endif diff --git a/lib/libspl/include/sys/types.h b/lib/libspl/include/sys/types.h index 8996af96ac23..c58b2d56600f 100644 --- a/lib/libspl/include/sys/types.h +++ b/lib/libspl/include/sys/types.h @@ -64,6 +64,7 @@ typedef int major_t; typedef int minor_t; typedef ushort_t o_mode_t; /* old file attribute type */ +typedef short index_t; /* * Definitions remaining from previous partial support for 64-bit file diff --git a/lib/libuutil/uu_misc.c b/lib/libuutil/uu_misc.c index 67f757c905a0..b10afd8eadf1 100644 --- a/lib/libuutil/uu_misc.c +++ b/lib/libuutil/uu_misc.c @@ -199,6 +199,8 @@ uu_panic(const char *format, ...) (void) vfprintf(stderr, format, args); + va_end(args); + if (uu_panic_thread == pthread_self()) abort(); else diff --git a/lib/libzfs/Makefile.am b/lib/libzfs/Makefile.am index 9a32a3013082..f1260ea7195d 100644 --- a/lib/libzfs/Makefile.am +++ b/lib/libzfs/Makefile.am @@ -15,7 +15,6 @@ USER_C = \ libzfs_dataset.c \ libzfs_diff.c \ libzfs_fru.c \ - libzfs_graph.c \ libzfs_import.c \ libzfs_iter.c \ libzfs_mount.c \ diff --git a/lib/libzfs/libzfs_changelist.c b/lib/libzfs/libzfs_changelist.c index 0bcfc0423b6b..5e007c0f4842 100644 --- a/lib/libzfs/libzfs_changelist.c +++ b/lib/libzfs/libzfs_changelist.c @@ -24,6 +24,7 @@ * Use is subject to license terms. * * Portions Copyright 2007 Ramprakash Jelari + * Copyright (c) 2014, 2015 by Delphix. All rights reserved. */ #include @@ -287,7 +288,7 @@ void changelist_rename(prop_changelist_t *clp, const char *src, const char *dst) { prop_changenode_t *cn; - char newname[ZFS_MAXNAMELEN]; + char newname[ZFS_MAX_DATASET_NAME_LEN]; for (cn = uu_list_first(clp->cl_list); cn != NULL; cn = uu_list_next(clp->cl_list, cn)) { diff --git a/lib/libzfs/libzfs_dataset.c b/lib/libzfs/libzfs_dataset.c index cc26927eb1ff..3ff96f0c65eb 100644 --- a/lib/libzfs/libzfs_dataset.c +++ b/lib/libzfs/libzfs_dataset.c @@ -43,7 +43,6 @@ #include #include #include -#include #include #include #include @@ -552,7 +551,7 @@ zfs_bookmark_exists(const char *path) { nvlist_t *bmarks; nvlist_t *props; - char fsname[ZFS_MAXNAMELEN]; + char fsname[ZFS_MAX_DATASET_NAME_LEN]; char *bmark_name; char *pound; int err; @@ -1474,6 +1473,7 @@ zfs_setprop_error(libzfs_handle_t *hdl, zfs_prop_t prop, int err, case ERANGE: if (prop == ZFS_PROP_COMPRESSION || + prop == ZFS_PROP_DNODESIZE || prop == ZFS_PROP_RECORDSIZE) { (void) zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "property setting is not allowed on " @@ -1865,22 +1865,21 @@ getprop_uint64(zfs_handle_t *zhp, zfs_prop_t prop, char **source) return (value); } -static char * +static const char * getprop_string(zfs_handle_t *zhp, zfs_prop_t prop, char **source) { nvlist_t *nv; - char *value; + const char *value; *source = NULL; if (nvlist_lookup_nvlist(zhp->zfs_props, zfs_prop_to_name(prop), &nv) == 0) { - verify(nvlist_lookup_string(nv, ZPROP_VALUE, &value) == 0); + value = fnvlist_lookup_string(nv, ZPROP_VALUE); (void) nvlist_lookup_string(nv, ZPROP_SOURCE, source); } else { verify(!zhp->zfs_props_table || zhp->zfs_props_table[prop] == B_TRUE); - if ((value = (char *)zfs_prop_default_string(prop)) == NULL) - value = ""; + value = zfs_prop_default_string(prop); *source = ""; } @@ -2202,7 +2201,7 @@ struct get_clones_arg { uint64_t numclones; nvlist_t *value; const char *origin; - char buf[ZFS_MAXNAMELEN]; + char buf[ZFS_MAX_DATASET_NAME_LEN]; }; int @@ -2257,7 +2256,7 @@ zfs_get_clones_nvl(zfs_handle_t *zhp) if (gca.numclones != 0) { zfs_handle_t *root; - char pool[ZFS_MAXNAMELEN]; + char pool[ZFS_MAX_DATASET_NAME_LEN]; char *cp = pool; /* get the pool name */ @@ -2301,7 +2300,7 @@ zfs_prop_get(zfs_handle_t *zhp, zfs_prop_t prop, char *propbuf, size_t proplen, { char *source = NULL; uint64_t val; - char *str; + const char *str; const char *strval; boolean_t received = zfs_is_recvd_props_mode(zhp); @@ -2407,14 +2406,10 @@ zfs_prop_get(zfs_handle_t *zhp, zfs_prop_t prop, char *propbuf, size_t proplen, break; case ZFS_PROP_ORIGIN: - (void) strlcpy(propbuf, getprop_string(zhp, prop, &source), - proplen); - /* - * If there is no parent at all, return failure to indicate that - * it doesn't apply to this dataset. - */ - if (propbuf[0] == '\0') + str = getprop_string(zhp, prop, &source); + if (str == NULL) return (-1); + (void) strlcpy(propbuf, str, proplen); break; case ZFS_PROP_CLONES: @@ -2596,8 +2591,10 @@ zfs_prop_get(zfs_handle_t *zhp, zfs_prop_t prop, char *propbuf, size_t proplen, break; case PROP_TYPE_STRING: - (void) strlcpy(propbuf, - getprop_string(zhp, prop, &source), proplen); + str = getprop_string(zhp, prop, &source); + if (str == NULL) + return (-1); + (void) strlcpy(propbuf, str, proplen); break; case PROP_TYPE_INDEX: @@ -3004,7 +3001,7 @@ check_parents(libzfs_handle_t *hdl, const char *path, uint64_t *zoned, boolean_t accept_ancestor, int *prefixlen) { zfs_cmd_t zc = {"\0"}; - char parent[ZFS_MAXNAMELEN]; + char parent[ZFS_MAX_DATASET_NAME_LEN]; char *slash; zfs_handle_t *zhp; char errbuf[1024]; @@ -3243,7 +3240,7 @@ zfs_create(libzfs_handle_t *hdl, const char *path, zfs_type_t type, ost = DMU_OST_ZFS; /* open zpool handle for prop validation */ - char pool_path[MAXNAMELEN]; + char pool_path[ZFS_MAX_DATASET_NAME_LEN]; (void) strlcpy(pool_path, path, sizeof (pool_path)); /* truncate pool_path at first slash */ @@ -3312,7 +3309,7 @@ zfs_create(libzfs_handle_t *hdl, const char *path, zfs_type_t type, /* check for failure */ if (ret != 0) { - char parent[ZFS_MAXNAMELEN]; + char parent[ZFS_MAX_DATASET_NAME_LEN]; (void) parent_name(path, parent, sizeof (parent)); switch (errno) { @@ -3402,7 +3399,7 @@ static int zfs_check_snap_cb(zfs_handle_t *zhp, void *arg) { struct destroydata *dd = arg; - char name[ZFS_MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; int rv = 0; (void) snprintf(name, sizeof (name), @@ -3493,7 +3490,7 @@ zfs_destroy_snaps_nvl(libzfs_handle_t *hdl, nvlist_t *snaps, boolean_t defer) int zfs_clone(zfs_handle_t *zhp, const char *target, nvlist_t *props) { - char parent[ZFS_MAXNAMELEN]; + char parent[ZFS_MAX_DATASET_NAME_LEN]; int ret; char errbuf[1024]; libzfs_handle_t *hdl = zhp->zfs_hdl; @@ -3623,7 +3620,7 @@ static int zfs_snapshot_cb(zfs_handle_t *zhp, void *arg) { snapdata_t *sd = arg; - char name[ZFS_MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; int rv = 0; if (zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) == 0) { @@ -3672,7 +3669,7 @@ zfs_snapshot_nvl(libzfs_handle_t *hdl, nvlist_t *snaps, nvlist_t *props) * get pool handle for prop validation. assumes all snaps are in the * same pool, as does lzc_snapshot (below). */ - char pool[MAXNAMELEN]; + char pool[ZFS_MAX_DATASET_NAME_LEN]; elem = nvlist_next_nvpair(snaps, NULL); (void) strlcpy(pool, nvpair_name(elem), sizeof (pool)); pool[strcspn(pool, "/@")] = '\0'; @@ -3726,7 +3723,7 @@ zfs_snapshot(libzfs_handle_t *hdl, const char *path, boolean_t recursive, { int ret; snapdata_t sd = { 0 }; - char fsname[ZFS_MAXNAMELEN]; + char fsname[ZFS_MAX_DATASET_NAME_LEN]; char *cp; zfs_handle_t *zhp; char errbuf[1024]; @@ -3905,7 +3902,7 @@ zfs_rename(zfs_handle_t *zhp, const char *target, boolean_t recursive, prop_changelist_t *cl = NULL; zfs_handle_t *zhrp = NULL; char *parentname = NULL; - char parent[ZFS_MAXNAMELEN]; + char parent[ZFS_MAX_DATASET_NAME_LEN]; libzfs_handle_t *hdl = zhp->zfs_hdl; char errbuf[1024]; @@ -4325,7 +4322,7 @@ zfs_userspace(zfs_handle_t *zhp, zfs_userquota_prop_t type, zc.zc_nvlist_dst_size = sizeof (buf); if (zfs_ioctl(hdl, ZFS_IOC_USERSPACE_MANY, &zc) != 0) { - char errbuf[ZFS_MAXNAMELEN + 32]; + char errbuf[1024]; (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, @@ -4359,7 +4356,7 @@ static int zfs_hold_one(zfs_handle_t *zhp, void *arg) { struct holdarg *ha = arg; - char name[ZFS_MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; int rv = 0; (void) snprintf(name, sizeof (name), @@ -4478,7 +4475,7 @@ static int zfs_release_one(zfs_handle_t *zhp, void *arg) { struct holdarg *ha = arg; - char name[ZFS_MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; int rv = 0; nvlist_t *existing_holds; @@ -4607,7 +4604,7 @@ zfs_get_fsacl(zfs_handle_t *zhp, nvlist_t **nvl) zc.zc_nvlist_dst_size = nvsz; zc.zc_nvlist_dst = (uintptr_t)nvbuf; - (void) strlcpy(zc.zc_name, zhp->zfs_name, ZFS_MAXNAMELEN); + (void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name)); if (ioctl(hdl->libzfs_fd, ZFS_IOC_GET_FSACL, &zc) != 0) { (void) snprintf(errbuf, sizeof (errbuf), diff --git a/lib/libzfs/libzfs_diff.c b/lib/libzfs/libzfs_diff.c index c03603934408..f5c799529c6e 100644 --- a/lib/libzfs/libzfs_diff.c +++ b/lib/libzfs/libzfs_diff.c @@ -22,6 +22,7 @@ /* * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2015 Nexenta Systems, Inc. All rights reserved. + * Copyright (c) 2015 by Delphix. All rights reserved. * Copyright 2016 Joyent, Inc. */ @@ -604,7 +605,7 @@ get_snapshot_names(differ_info_t *di, const char *fromsnap, * not the same dataset name, might be okay if * tosnap is a clone of a fromsnap descendant. */ - char origin[ZFS_MAXNAMELEN]; + char origin[ZFS_MAX_DATASET_NAME_LEN]; zprop_source_t src; zfs_handle_t *zhp; diff --git a/lib/libzfs/libzfs_graph.c b/lib/libzfs/libzfs_graph.c deleted file mode 100644 index 63d9138ef33f..000000000000 --- a/lib/libzfs/libzfs_graph.c +++ /dev/null @@ -1,653 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2009 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -/* - * Iterate over all children of the current object. This includes the normal - * dataset hierarchy, but also arbitrary hierarchies due to clones. We want to - * walk all datasets in the pool, and construct a directed graph of the form: - * - * home - * | - * +----+----+ - * | | - * v v ws - * bar baz | - * | | - * v v - * @yesterday ----> foo - * - * In order to construct this graph, we have to walk every dataset in the pool, - * because the clone parent is stored as a property of the child, not the - * parent. The parent only keeps track of the number of clones. - * - * In the normal case (without clones) this would be rather expensive. To avoid - * unnecessary computation, we first try a walk of the subtree hierarchy - * starting from the initial node. At each dataset, we construct a node in the - * graph and an edge leading from its parent. If we don't see any snapshots - * with a non-zero clone count, then we are finished. - * - * If we do find a cloned snapshot, then we finish the walk of the current - * subtree, but indicate that we need to do a complete walk. We then perform a - * global walk of all datasets, avoiding the subtree we already processed. - * - * At the end of this, we'll end up with a directed graph of all relevant (and - * possible some irrelevant) datasets in the system. We need to both find our - * limiting subgraph and determine a safe ordering in which to destroy the - * datasets. We do a topological ordering of our graph starting at our target - * dataset, and then walk the results in reverse. - * - * It's possible for the graph to have cycles if, for example, the user renames - * a clone to be the parent of its origin snapshot. The user can request to - * generate an error in this case, or ignore the cycle and continue. - * - * When removing datasets, we want to destroy the snapshots in chronological - * order (because this is the most efficient method). In order to accomplish - * this, we store the creation transaction group with each vertex and keep each - * vertex's edges sorted according to this value. The topological sort will - * automatically walk the snapshots in the correct order. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "libzfs_impl.h" -#include "zfs_namecheck.h" - -#define MIN_EDGECOUNT 4 - -/* - * Vertex structure. Indexed by dataset name, this structure maintains a list - * of edges to other vertices. - */ -struct zfs_edge; -typedef struct zfs_vertex { - char zv_dataset[ZFS_MAXNAMELEN]; - struct zfs_vertex *zv_next; - int zv_visited; - uint64_t zv_txg; - struct zfs_edge **zv_edges; - int zv_edgecount; - int zv_edgealloc; -} zfs_vertex_t; - -enum { - VISIT_SEEN = 1, - VISIT_SORT_PRE, - VISIT_SORT_POST -}; - -/* - * Edge structure. Simply maintains a pointer to the destination vertex. There - * is no need to store the source vertex, since we only use edges in the context - * of the source vertex. - */ -typedef struct zfs_edge { - zfs_vertex_t *ze_dest; - struct zfs_edge *ze_next; -} zfs_edge_t; - -#define ZFS_GRAPH_SIZE 1027 /* this could be dynamic some day */ - -/* - * Graph structure. Vertices are maintained in a hash indexed by dataset name. - */ -typedef struct zfs_graph { - zfs_vertex_t **zg_hash; - size_t zg_size; - size_t zg_nvertex; - const char *zg_root; - int zg_clone_count; -} zfs_graph_t; - -/* - * Allocate a new edge pointing to the target vertex. - */ -static zfs_edge_t * -zfs_edge_create(libzfs_handle_t *hdl, zfs_vertex_t *dest) -{ - zfs_edge_t *zep = zfs_alloc(hdl, sizeof (zfs_edge_t)); - - if (zep == NULL) - return (NULL); - - zep->ze_dest = dest; - - return (zep); -} - -/* - * Destroy an edge. - */ -static void -zfs_edge_destroy(zfs_edge_t *zep) -{ - free(zep); -} - -/* - * Allocate a new vertex with the given name. - */ -static zfs_vertex_t * -zfs_vertex_create(libzfs_handle_t *hdl, const char *dataset) -{ - zfs_vertex_t *zvp = zfs_alloc(hdl, sizeof (zfs_vertex_t)); - - if (zvp == NULL) - return (NULL); - - assert(strlen(dataset) < ZFS_MAXNAMELEN); - - (void) strlcpy(zvp->zv_dataset, dataset, sizeof (zvp->zv_dataset)); - - if ((zvp->zv_edges = zfs_alloc(hdl, - MIN_EDGECOUNT * sizeof (void *))) == NULL) { - free(zvp); - return (NULL); - } - - zvp->zv_edgealloc = MIN_EDGECOUNT; - - return (zvp); -} - -/* - * Destroy a vertex. Frees up any associated edges. - */ -static void -zfs_vertex_destroy(zfs_vertex_t *zvp) -{ - int i; - - for (i = 0; i < zvp->zv_edgecount; i++) - zfs_edge_destroy(zvp->zv_edges[i]); - - free(zvp->zv_edges); - free(zvp); -} - -/* - * Given a vertex, add an edge to the destination vertex. - */ -static int -zfs_vertex_add_edge(libzfs_handle_t *hdl, zfs_vertex_t *zvp, - zfs_vertex_t *dest) -{ - zfs_edge_t *zep = zfs_edge_create(hdl, dest); - - if (zep == NULL) - return (-1); - - if (zvp->zv_edgecount == zvp->zv_edgealloc) { - void *ptr; - - if ((ptr = zfs_realloc(hdl, zvp->zv_edges, - zvp->zv_edgealloc * sizeof (void *), - zvp->zv_edgealloc * 2 * sizeof (void *))) == NULL) - return (-1); - - zvp->zv_edges = ptr; - zvp->zv_edgealloc *= 2; - } - - zvp->zv_edges[zvp->zv_edgecount++] = zep; - - return (0); -} - -static int -zfs_edge_compare(const void *a, const void *b) -{ - const zfs_edge_t *ea = *((zfs_edge_t **)a); - const zfs_edge_t *eb = *((zfs_edge_t **)b); - - if (ea->ze_dest->zv_txg < eb->ze_dest->zv_txg) - return (-1); - if (ea->ze_dest->zv_txg > eb->ze_dest->zv_txg) - return (1); - return (0); -} - -/* - * Sort the given vertex edges according to the creation txg of each vertex. - */ -static void -zfs_vertex_sort_edges(zfs_vertex_t *zvp) -{ - if (zvp->zv_edgecount == 0) - return; - - qsort(zvp->zv_edges, zvp->zv_edgecount, sizeof (void *), - zfs_edge_compare); -} - -/* - * Construct a new graph object. We allow the size to be specified as a - * parameter so in the future we can size the hash according to the number of - * datasets in the pool. - */ -static zfs_graph_t * -zfs_graph_create(libzfs_handle_t *hdl, const char *dataset, size_t size) -{ - zfs_graph_t *zgp = zfs_alloc(hdl, sizeof (zfs_graph_t)); - - if (zgp == NULL) - return (NULL); - - zgp->zg_size = size; - if ((zgp->zg_hash = zfs_alloc(hdl, - size * sizeof (zfs_vertex_t *))) == NULL) { - free(zgp); - return (NULL); - } - - zgp->zg_root = dataset; - zgp->zg_clone_count = 0; - - return (zgp); -} - -/* - * Destroy a graph object. We have to iterate over all the hash chains, - * destroying each vertex in the process. - */ -static void -zfs_graph_destroy(zfs_graph_t *zgp) -{ - int i; - zfs_vertex_t *current, *next; - - for (i = 0; i < zgp->zg_size; i++) { - current = zgp->zg_hash[i]; - while (current != NULL) { - next = current->zv_next; - zfs_vertex_destroy(current); - current = next; - } - } - - free(zgp->zg_hash); - free(zgp); -} - -/* - * Graph hash function. Classic bernstein k=33 hash function, taken from - * usr/src/cmd/sgs/tools/common/strhash.c - */ -static size_t -zfs_graph_hash(zfs_graph_t *zgp, const char *str) -{ - size_t hash = 5381; - int c; - - while ((c = *str++) != 0) - hash = ((hash << 5) + hash) + c; /* hash * 33 + c */ - - return (hash % zgp->zg_size); -} - -/* - * Given a dataset name, finds the associated vertex, creating it if necessary. - */ -static zfs_vertex_t * -zfs_graph_lookup(libzfs_handle_t *hdl, zfs_graph_t *zgp, const char *dataset, - uint64_t txg) -{ - size_t idx = zfs_graph_hash(zgp, dataset); - zfs_vertex_t *zvp; - - for (zvp = zgp->zg_hash[idx]; zvp != NULL; zvp = zvp->zv_next) { - if (strcmp(zvp->zv_dataset, dataset) == 0) { - if (zvp->zv_txg == 0) - zvp->zv_txg = txg; - return (zvp); - } - } - - if ((zvp = zfs_vertex_create(hdl, dataset)) == NULL) - return (NULL); - - zvp->zv_next = zgp->zg_hash[idx]; - zvp->zv_txg = txg; - zgp->zg_hash[idx] = zvp; - zgp->zg_nvertex++; - - return (zvp); -} - -/* - * Given two dataset names, create an edge between them. For the source vertex, - * mark 'zv_visited' to indicate that we have seen this vertex, and not simply - * created it as a destination of another edge. If 'dest' is NULL, then this - * is an individual vertex (i.e. the starting vertex), so don't add an edge. - */ -static int -zfs_graph_add(libzfs_handle_t *hdl, zfs_graph_t *zgp, const char *source, - const char *dest, uint64_t txg) -{ - zfs_vertex_t *svp, *dvp; - - if ((svp = zfs_graph_lookup(hdl, zgp, source, 0)) == NULL) - return (-1); - svp->zv_visited = VISIT_SEEN; - if (dest != NULL) { - dvp = zfs_graph_lookup(hdl, zgp, dest, txg); - if (dvp == NULL) - return (-1); - if (zfs_vertex_add_edge(hdl, svp, dvp) != 0) - return (-1); - } - - return (0); -} - -/* - * Iterate over all children of the given dataset, adding any vertices - * as necessary. Returns -1 if there was an error, or 0 otherwise. - * This is a simple recursive algorithm - the ZFS namespace typically - * is very flat. We manually invoke the necessary ioctl() calls to - * avoid the overhead and additional semantics of zfs_open(). - */ -static int -iterate_children(libzfs_handle_t *hdl, zfs_graph_t *zgp, const char *dataset) -{ - zfs_cmd_t zc = {"\0"}; - zfs_vertex_t *zvp; - - /* - * Look up the source vertex, and avoid it if we've seen it before. - */ - zvp = zfs_graph_lookup(hdl, zgp, dataset, 0); - if (zvp == NULL) - return (-1); - if (zvp->zv_visited == VISIT_SEEN) - return (0); - - /* - * Iterate over all children - */ - for ((void) strlcpy(zc.zc_name, dataset, sizeof (zc.zc_name)); - ioctl(hdl->libzfs_fd, ZFS_IOC_DATASET_LIST_NEXT, &zc) == 0; - (void) strlcpy(zc.zc_name, dataset, sizeof (zc.zc_name))) { - /* - * Get statistics for this dataset, to determine the type of the - * dataset and clone statistics. If this fails, the dataset has - * since been removed, and we're pretty much screwed anyway. - */ - zc.zc_objset_stats.dds_origin[0] = '\0'; - if (ioctl(hdl->libzfs_fd, ZFS_IOC_OBJSET_STATS, &zc) != 0) - continue; - - if (zc.zc_objset_stats.dds_origin[0] != '\0') { - if (zfs_graph_add(hdl, zgp, - zc.zc_objset_stats.dds_origin, zc.zc_name, - zc.zc_objset_stats.dds_creation_txg) != 0) - return (-1); - /* - * Count origins only if they are contained in the graph - */ - if (isa_child_of(zc.zc_objset_stats.dds_origin, - zgp->zg_root)) - zgp->zg_clone_count--; - } - - /* - * Add an edge between the parent and the child. - */ - if (zfs_graph_add(hdl, zgp, dataset, zc.zc_name, - zc.zc_objset_stats.dds_creation_txg) != 0) - return (-1); - - /* - * Recursively visit child - */ - if (iterate_children(hdl, zgp, zc.zc_name)) - return (-1); - } - - /* - * Now iterate over all snapshots. - */ - bzero(&zc, sizeof (zc)); - - for ((void) strlcpy(zc.zc_name, dataset, sizeof (zc.zc_name)); - ioctl(hdl->libzfs_fd, ZFS_IOC_SNAPSHOT_LIST_NEXT, &zc) == 0; - (void) strlcpy(zc.zc_name, dataset, sizeof (zc.zc_name))) { - - /* - * Get statistics for this dataset, to determine the type of the - * dataset and clone statistics. If this fails, the dataset has - * since been removed, and we're pretty much screwed anyway. - */ - if (ioctl(hdl->libzfs_fd, ZFS_IOC_OBJSET_STATS, &zc) != 0) - continue; - - /* - * Add an edge between the parent and the child. - */ - if (zfs_graph_add(hdl, zgp, dataset, zc.zc_name, - zc.zc_objset_stats.dds_creation_txg) != 0) - return (-1); - - zgp->zg_clone_count += zc.zc_objset_stats.dds_num_clones; - } - - zvp->zv_visited = VISIT_SEEN; - - return (0); -} - -/* - * Returns false if there are no snapshots with dependent clones in this - * subtree or if all of those clones are also in this subtree. Returns - * true if there is an error or there are external dependents. - */ -static boolean_t -external_dependents(libzfs_handle_t *hdl, zfs_graph_t *zgp, const char *dataset) -{ - zfs_cmd_t zc = {"\0"}; - - /* - * Check whether this dataset is a clone or has clones since - * iterate_children() only checks the children. - */ - (void) strlcpy(zc.zc_name, dataset, sizeof (zc.zc_name)); - if (ioctl(hdl->libzfs_fd, ZFS_IOC_OBJSET_STATS, &zc) != 0) - return (B_TRUE); - - if (zc.zc_objset_stats.dds_origin[0] != '\0') { - if (zfs_graph_add(hdl, zgp, - zc.zc_objset_stats.dds_origin, zc.zc_name, - zc.zc_objset_stats.dds_creation_txg) != 0) - return (B_TRUE); - if (isa_child_of(zc.zc_objset_stats.dds_origin, dataset)) - zgp->zg_clone_count--; - } - - if ((zc.zc_objset_stats.dds_num_clones) || - iterate_children(hdl, zgp, dataset)) - return (B_TRUE); - - return (zgp->zg_clone_count != 0); -} - -/* - * Construct a complete graph of all necessary vertices. First, iterate over - * only our object's children. If no cloned snapshots are found, or all of - * the cloned snapshots are in this subtree then return a graph of the subtree. - * Otherwise, start at the root of the pool and iterate over all datasets. - */ -static zfs_graph_t * -construct_graph(libzfs_handle_t *hdl, const char *dataset) -{ - zfs_graph_t *zgp = zfs_graph_create(hdl, dataset, ZFS_GRAPH_SIZE); - int ret = 0; - - if (zgp == NULL) - return (zgp); - - if ((strchr(dataset, '/') == NULL) || - (external_dependents(hdl, zgp, dataset))) { - /* - * Determine pool name and try again. - */ - int len = strcspn(dataset, "/@") + 1; - char *pool = zfs_alloc(hdl, len); - - if (pool == NULL) { - zfs_graph_destroy(zgp); - return (NULL); - } - (void) strlcpy(pool, dataset, len); - - if (iterate_children(hdl, zgp, pool) == -1 || - zfs_graph_add(hdl, zgp, pool, NULL, 0) != 0) { - free(pool); - zfs_graph_destroy(zgp); - return (NULL); - } - free(pool); - } - - if (ret == -1 || zfs_graph_add(hdl, zgp, dataset, NULL, 0) != 0) { - zfs_graph_destroy(zgp); - return (NULL); - } - - return (zgp); -} - -/* - * Given a graph, do a recursive topological sort into the given array. This is - * really just a depth first search, so that the deepest nodes appear first. - * hijack the 'zv_visited' marker to avoid visiting the same vertex twice. - */ -static int -topo_sort(libzfs_handle_t *hdl, boolean_t allowrecursion, char **result, - size_t *idx, zfs_vertex_t *zgv) -{ - int i; - - if (zgv->zv_visited == VISIT_SORT_PRE && !allowrecursion) { - /* - * If we've already seen this vertex as part of our depth-first - * search, then we have a cyclic dependency, and we must return - * an error. - */ - zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "recursive dependency at '%s'"), - zgv->zv_dataset); - return (zfs_error(hdl, EZFS_RECURSIVE, - dgettext(TEXT_DOMAIN, - "cannot determine dependent datasets"))); - } else if (zgv->zv_visited >= VISIT_SORT_PRE) { - /* - * If we've already processed this as part of the topological - * sort, then don't bother doing so again. - */ - return (0); - } - - zgv->zv_visited = VISIT_SORT_PRE; - - /* avoid doing a search if we don't have to */ - zfs_vertex_sort_edges(zgv); - for (i = 0; i < zgv->zv_edgecount; i++) { - if (topo_sort(hdl, allowrecursion, result, idx, - zgv->zv_edges[i]->ze_dest) != 0) - return (-1); - } - - /* we may have visited this in the course of the above */ - if (zgv->zv_visited == VISIT_SORT_POST) - return (0); - - if ((result[*idx] = zfs_alloc(hdl, - strlen(zgv->zv_dataset) + 1)) == NULL) - return (-1); - - (void) strcpy(result[*idx], zgv->zv_dataset); - *idx += 1; - zgv->zv_visited = VISIT_SORT_POST; - return (0); -} - -/* - * The only public interface for this file. Do the dirty work of constructing a - * child list for the given object. Construct the graph, do the toplogical - * sort, and then return the array of strings to the caller. - * - * The 'allowrecursion' parameter controls behavior when cycles are found. If - * it is set, the the cycle is ignored and the results returned as if the cycle - * did not exist. If it is not set, then the routine will generate an error if - * a cycle is found. - */ -int -get_dependents(libzfs_handle_t *hdl, boolean_t allowrecursion, - const char *dataset, char ***result, size_t *count) -{ - zfs_graph_t *zgp; - zfs_vertex_t *zvp; - - if ((zgp = construct_graph(hdl, dataset)) == NULL) - return (-1); - - if ((*result = zfs_alloc(hdl, - zgp->zg_nvertex * sizeof (char *))) == NULL) { - zfs_graph_destroy(zgp); - return (-1); - } - - if ((zvp = zfs_graph_lookup(hdl, zgp, dataset, 0)) == NULL) { - free(*result); - zfs_graph_destroy(zgp); - return (-1); - } - - *count = 0; - if (topo_sort(hdl, allowrecursion, *result, count, zvp) != 0) { - free(*result); - zfs_graph_destroy(zgp); - return (-1); - } - - /* - * Get rid of the last entry, which is our starting vertex and not - * strictly a dependent. - */ - assert(*count > 0); - free((*result)[*count - 1]); - (*count)--; - - zfs_graph_destroy(zgp); - - return (0); -} diff --git a/lib/libzfs/libzfs_import.c b/lib/libzfs/libzfs_import.c index 99c3d9998f23..fb3525848c94 100644 --- a/lib/libzfs/libzfs_import.c +++ b/lib/libzfs/libzfs_import.c @@ -760,116 +760,6 @@ add_config(libzfs_handle_t *hdl, pool_list_t *pl, const char *path, return (0); } -static int -add_path(libzfs_handle_t *hdl, pool_list_t *pools, uint64_t pool_guid, - uint64_t vdev_guid, const char *path, int order) -{ - nvlist_t *label; - uint64_t guid; - int error, fd, num_labels; - - fd = open64(path, O_RDONLY); - if (fd < 0) - return (errno); - - error = zpool_read_label(fd, &label, &num_labels); - close(fd); - - if (error || label == NULL) - return (ENOENT); - - error = nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID, &guid); - if (error || guid != pool_guid) { - nvlist_free(label); - return (EINVAL); - } - - error = nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &guid); - if (error || guid != vdev_guid) { - nvlist_free(label); - return (EINVAL); - } - - error = add_config(hdl, pools, path, order, num_labels, label); - - return (error); -} - -static int -add_configs_from_label_impl(libzfs_handle_t *hdl, pool_list_t *pools, - nvlist_t *nvroot, uint64_t pool_guid, uint64_t vdev_guid) -{ - char udevpath[MAXPATHLEN]; - char *path; - nvlist_t **child; - uint_t c, children; - uint64_t guid; - int error; - - if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, - &child, &children) == 0) { - for (c = 0; c < children; c++) { - error = add_configs_from_label_impl(hdl, pools, - child[c], pool_guid, vdev_guid); - if (error) - return (error); - } - return (0); - } - - if (nvroot == NULL) - return (0); - - error = nvlist_lookup_uint64(nvroot, ZPOOL_CONFIG_GUID, &guid); - if ((error != 0) || (guid != vdev_guid)) - return (0); - - error = nvlist_lookup_string(nvroot, ZPOOL_CONFIG_PATH, &path); - if (error == 0) - (void) add_path(hdl, pools, pool_guid, vdev_guid, path, 0); - - error = nvlist_lookup_string(nvroot, ZPOOL_CONFIG_DEVID, &path); - if (error == 0) { - sprintf(udevpath, "%s%s", DEV_BYID_PATH, path); - (void) add_path(hdl, pools, pool_guid, vdev_guid, udevpath, 1); - } - - return (0); -} - -/* - * Given a disk label call add_config() for all known paths to the device - * as described by the label itself. The paths are added in the following - * priority order: 'path', 'devid', 'devnode'. As these alternate paths are - * added the labels are verified to make sure they refer to the same device. - */ -static int -add_configs_from_label(libzfs_handle_t *hdl, pool_list_t *pools, - char *devname, int num_labels, nvlist_t *label) -{ - nvlist_t *nvroot; - uint64_t pool_guid; - uint64_t vdev_guid; - int error; - - if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_VDEV_TREE, &nvroot) || - nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID, &pool_guid) || - nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &vdev_guid)) - return (ENOENT); - - /* Allow devlinks to stabilize so all paths are available. */ - zpool_label_disk_wait(devname, DISK_LABEL_WAIT); - - /* Add alternate paths as described by the label vdev_tree. */ - (void) add_configs_from_label_impl(hdl, pools, nvroot, - pool_guid, vdev_guid); - - /* Add the device node /dev/sdX path as a last resort. */ - error = add_config(hdl, pools, devname, 100, num_labels, label); - - return (error); -} - /* * Returns true if the named pool matches the given GUID. */ @@ -1454,14 +1344,15 @@ zpool_read_label(int fd, nvlist_t **config, int *num_labels) } typedef struct rdsk_node { - char *rn_name; - int rn_num_labels; - int rn_dfd; + char *rn_name; /* Full path to device */ + int rn_order; /* Preferred order (low to high) */ + int rn_num_labels; /* Number of valid labels */ libzfs_handle_t *rn_hdl; - nvlist_t *rn_config; + nvlist_t *rn_config; /* Label config */ avl_tree_t *rn_avl; avl_node_t rn_node; - boolean_t rn_nozpool; + kmutex_t *rn_lock; + boolean_t rn_labelpaths; } rdsk_node_t; static int @@ -1499,83 +1390,6 @@ slice_cache_compare(const void *arg1, const void *arg2) return (rv > 0 ? 1 : -1); } -#ifndef __linux__ -static void -check_one_slice(avl_tree_t *r, char *diskname, uint_t partno, - diskaddr_t size, uint_t blksz) -{ - rdsk_node_t tmpnode; - rdsk_node_t *node; - char sname[MAXNAMELEN]; - - tmpnode.rn_name = &sname[0]; - (void) snprintf(tmpnode.rn_name, MAXNAMELEN, "%s%u", - diskname, partno); - /* too small to contain a zpool? */ - if ((size < (SPA_MINDEVSIZE / blksz)) && - (node = avl_find(r, &tmpnode, NULL))) - node->rn_nozpool = B_TRUE; -} -#endif - -static void -nozpool_all_slices(avl_tree_t *r, const char *sname) -{ -#ifndef __linux__ - char diskname[MAXNAMELEN]; - char *ptr; - int i; - - (void) strncpy(diskname, sname, MAXNAMELEN); - if (((ptr = strrchr(diskname, 's')) == NULL) && - ((ptr = strrchr(diskname, 'p')) == NULL)) - return; - ptr[0] = 's'; - ptr[1] = '\0'; - for (i = 0; i < NDKMAP; i++) - check_one_slice(r, diskname, i, 0, 1); - ptr[0] = 'p'; - for (i = 0; i <= FD_NUMPART; i++) - check_one_slice(r, diskname, i, 0, 1); -#endif -} - -static void -check_slices(avl_tree_t *r, int fd, const char *sname) -{ -#ifndef __linux__ - struct extvtoc vtoc; - struct dk_gpt *gpt; - char diskname[MAXNAMELEN]; - char *ptr; - int i; - - (void) strncpy(diskname, sname, MAXNAMELEN); - if ((ptr = strrchr(diskname, 's')) == NULL || !isdigit(ptr[1])) - return; - ptr[1] = '\0'; - - if (read_extvtoc(fd, &vtoc) >= 0) { - for (i = 0; i < NDKMAP; i++) - check_one_slice(r, diskname, i, - vtoc.v_part[i].p_size, vtoc.v_sectorsz); - } else if (efi_alloc_and_read(fd, &gpt) >= 0) { - /* - * on x86 we'll still have leftover links that point - * to slices s[9-15], so use NDKMAP instead - */ - for (i = 0; i < NDKMAP; i++) - check_one_slice(r, diskname, i, - gpt->efi_parts[i].p_size, gpt->efi_lbasize); - /* nodes p[1-4] are never used with EFI labels */ - ptr[0] = 'p'; - for (i = 1; i <= FD_NUMPART; i++) - check_one_slice(r, diskname, i, 0, 1); - efi_free(gpt); - } -#endif -} - static boolean_t is_watchdog_dev(char *dev) { @@ -1590,18 +1404,81 @@ is_watchdog_dev(char *dev) return (B_FALSE); } +static int +label_paths_impl(libzfs_handle_t *hdl, nvlist_t *nvroot, uint64_t pool_guid, + uint64_t vdev_guid, char **path, char **devid) +{ + nvlist_t **child; + uint_t c, children; + uint64_t guid; + char *val; + int error; + + if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_CHILDREN, + &child, &children) == 0) { + for (c = 0; c < children; c++) { + error = label_paths_impl(hdl, child[c], + pool_guid, vdev_guid, path, devid); + if (error) + return (error); + } + return (0); + } + + if (nvroot == NULL) + return (0); + + error = nvlist_lookup_uint64(nvroot, ZPOOL_CONFIG_GUID, &guid); + if ((error != 0) || (guid != vdev_guid)) + return (0); + + error = nvlist_lookup_string(nvroot, ZPOOL_CONFIG_PATH, &val); + if (error == 0) + *path = val; + + error = nvlist_lookup_string(nvroot, ZPOOL_CONFIG_DEVID, &val); + if (error == 0) + *devid = val; + + return (0); +} + +/* + * Given a disk label fetch the ZPOOL_CONFIG_PATH and ZPOOL_CONFIG_DEVID + * and store these strings as config_path and devid_path respectively. + * The returned pointers are only valid as long as label remains valid. + */ +static int +label_paths(libzfs_handle_t *hdl, nvlist_t *label, char **path, char **devid) +{ + nvlist_t *nvroot; + uint64_t pool_guid; + uint64_t vdev_guid; + + *path = NULL; + *devid = NULL; + + if (nvlist_lookup_nvlist(label, ZPOOL_CONFIG_VDEV_TREE, &nvroot) || + nvlist_lookup_uint64(label, ZPOOL_CONFIG_POOL_GUID, &pool_guid) || + nvlist_lookup_uint64(label, ZPOOL_CONFIG_GUID, &vdev_guid)) + return (ENOENT); + + return (label_paths_impl(hdl, nvroot, pool_guid, vdev_guid, path, + devid)); +} + static void zpool_open_func(void *arg) { rdsk_node_t *rn = arg; + libzfs_handle_t *hdl = rn->rn_hdl; struct stat64 statbuf; nvlist_t *config; + char *bname, *dupname; + int error; int num_labels; int fd; - if (rn->rn_nozpool) - return; -#ifdef __linux__ /* * Skip devices with well known prefixes there can be side effects * when opening devices which need to be avoided. @@ -1609,58 +1486,34 @@ zpool_open_func(void *arg) * hpet - High Precision Event Timer * watchdog - Watchdog must be closed in a special way. */ - if ((strcmp(rn->rn_name, "hpet") == 0) || - is_watchdog_dev(rn->rn_name)) + dupname = zfs_strdup(hdl, rn->rn_name); + bname = basename(dupname); + error = ((strcmp(bname, "hpet") == 0) || is_watchdog_dev(bname)); + free(dupname); + if (error) return; /* * Ignore failed stats. We only want regular files and block devices. */ - if (fstatat64(rn->rn_dfd, rn->rn_name, &statbuf, 0) != 0 || + if (stat64(rn->rn_name, &statbuf) != 0 || (!S_ISREG(statbuf.st_mode) && !S_ISBLK(statbuf.st_mode))) return; - if ((fd = openat64(rn->rn_dfd, rn->rn_name, O_RDONLY)) < 0) { - /* symlink to a device that's no longer there */ - if (errno == ENOENT) - nozpool_all_slices(rn->rn_avl, rn->rn_name); + if ((fd = open(rn->rn_name, O_RDONLY)) < 0) return; - } -#else - if ((fd = openat64(rn->rn_dfd, rn->rn_name, O_RDONLY)) < 0) { - /* symlink to a device that's no longer there */ - if (errno == ENOENT) - nozpool_all_slices(rn->rn_avl, rn->rn_name); - return; - } + /* - * Ignore failed stats. We only want regular - * files, character devs and block devs. + * This file is too small to hold a zpool */ - if (fstat64(fd, &statbuf) != 0 || - (!S_ISREG(statbuf.st_mode) && - !S_ISCHR(statbuf.st_mode) && - !S_ISBLK(statbuf.st_mode))) { - (void) close(fd); - return; - } -#endif - /* this file is too small to hold a zpool */ if (S_ISREG(statbuf.st_mode) && statbuf.st_size < SPA_MINDEVSIZE) { (void) close(fd); return; - } else if (!S_ISREG(statbuf.st_mode)) { - /* - * Try to read the disk label first so we don't have to - * open a bunch of minor nodes that can't have a zpool. - */ - check_slices(rn->rn_avl, fd, rn->rn_name); } if ((zpool_read_label(fd, &config, &num_labels)) != 0) { (void) close(fd); - (void) no_memory(rn->rn_hdl); return; } @@ -1674,6 +1527,69 @@ zpool_open_func(void *arg) rn->rn_config = config; rn->rn_num_labels = num_labels; + + /* + * Add additional entries for paths described by this label. + */ + if (rn->rn_labelpaths) { + char *path = NULL; + char *devid = NULL; + rdsk_node_t *slice; + avl_index_t where; + int error; + + if (label_paths(rn->rn_hdl, rn->rn_config, &path, &devid)) + return; + + /* + * Allow devlinks to stabilize so all paths are available. + */ + zpool_label_disk_wait(rn->rn_name, DISK_LABEL_WAIT); + + if (path != NULL) { + slice = zfs_alloc(hdl, sizeof (rdsk_node_t)); + slice->rn_name = zfs_strdup(hdl, path); + slice->rn_avl = rn->rn_avl; + slice->rn_hdl = hdl; + slice->rn_order = 1; + slice->rn_labelpaths = B_FALSE; + mutex_enter(rn->rn_lock); + if (avl_find(rn->rn_avl, slice, &where)) { + mutex_exit(rn->rn_lock); + free(slice->rn_name); + free(slice); + } else { + avl_insert(rn->rn_avl, slice, where); + mutex_exit(rn->rn_lock); + zpool_open_func(slice); + } + } + + if (devid != NULL) { + slice = zfs_alloc(hdl, sizeof (rdsk_node_t)); + error = asprintf(&slice->rn_name, "%s%s", + DEV_BYID_PATH, devid); + if (error == -1) { + free(slice); + return; + } + + slice->rn_avl = rn->rn_avl; + slice->rn_hdl = hdl; + slice->rn_order = 2; + slice->rn_labelpaths = B_FALSE; + mutex_enter(rn->rn_lock); + if (avl_find(rn->rn_avl, slice, &where)) { + mutex_exit(rn->rn_lock); + free(slice->rn_name); + free(slice); + } else { + avl_insert(rn->rn_avl, slice, where); + mutex_exit(rn->rn_lock); + zpool_open_func(slice); + } + } + } } /* @@ -1709,69 +1625,145 @@ zpool_clear_label(int fd) } /* - * Use libblkid to quickly search for zfs devices + * Scan a list of directories for zfs devices. + */ +static int +zpool_find_import_scan(libzfs_handle_t *hdl, kmutex_t *lock, + avl_tree_t **slice_cache, char **dir, int dirs) +{ + avl_tree_t *cache; + rdsk_node_t *slice; + void *cookie; + int i, error; + + *slice_cache = NULL; + cache = zfs_alloc(hdl, sizeof (avl_tree_t)); + avl_create(cache, slice_cache_compare, sizeof (rdsk_node_t), + offsetof(rdsk_node_t, rn_node)); + + for (i = 0; i < dirs; i++) { + char path[MAXPATHLEN]; + struct dirent64 *dp; + DIR *dirp; + + if (realpath(dir[i], path) == NULL) { + error = errno; + if (error == ENOENT) + continue; + + zfs_error_aux(hdl, strerror(error)); + (void) zfs_error_fmt(hdl, EZFS_BADPATH, dgettext( + TEXT_DOMAIN, "cannot resolve path '%s'"), dir[i]); + goto error; + } + + dirp = opendir(path); + if (dirp == NULL) { + error = errno; + zfs_error_aux(hdl, strerror(error)); + (void) zfs_error_fmt(hdl, EZFS_BADPATH, + dgettext(TEXT_DOMAIN, "cannot open '%s'"), path); + goto error; + } + + while ((dp = readdir64(dirp)) != NULL) { + const char *name = dp->d_name; + if (name[0] == '.' && + (name[1] == 0 || (name[1] == '.' && name[2] == 0))) + continue; + + slice = zfs_alloc(hdl, sizeof (rdsk_node_t)); + error = asprintf(&slice->rn_name, "%s/%s", path, name); + if (error == -1) { + free(slice); + continue; + } + slice->rn_lock = lock; + slice->rn_avl = cache; + slice->rn_hdl = hdl; + slice->rn_order = i+1; + slice->rn_labelpaths = B_FALSE; + mutex_enter(lock); + avl_add(cache, slice); + mutex_exit(lock); + } + + (void) closedir(dirp); + } + + *slice_cache = cache; + return (0); + +error: + cookie = NULL; + while ((slice = avl_destroy_nodes(cache, &cookie)) != NULL) { + free(slice->rn_name); + free(slice); + } + free(cache); + + return (error); +} + +/* + * Use libblkid to quickly enumerate all known zfs devices. */ static int -zpool_find_import_blkid(libzfs_handle_t *hdl, pool_list_t *pools) +zpool_find_import_blkid(libzfs_handle_t *hdl, kmutex_t *lock, + avl_tree_t **slice_cache) { + rdsk_node_t *slice; blkid_cache cache; blkid_dev_iterate iter; blkid_dev dev; - int err; + int error; - err = blkid_get_cache(&cache, NULL); - if (err != 0) { - (void) zfs_error_fmt(hdl, EZFS_BADCACHE, - dgettext(TEXT_DOMAIN, "blkid_get_cache() %d"), err); - goto err_blkid1; - } + *slice_cache = NULL; + + error = blkid_get_cache(&cache, NULL); + if (error != 0) + return (error); - err = blkid_probe_all(cache); - if (err != 0) { - (void) zfs_error_fmt(hdl, EZFS_BADCACHE, - dgettext(TEXT_DOMAIN, "blkid_probe_all() %d"), err); - goto err_blkid2; + error = blkid_probe_all_new(cache); + if (error != 0) { + blkid_put_cache(cache); + return (error); } iter = blkid_dev_iterate_begin(cache); if (iter == NULL) { - (void) zfs_error_fmt(hdl, EZFS_BADCACHE, - dgettext(TEXT_DOMAIN, "blkid_dev_iterate_begin()")); - goto err_blkid2; + blkid_put_cache(cache); + return (EINVAL); } - err = blkid_dev_set_search(iter, "TYPE", "zfs_member"); - if (err != 0) { - (void) zfs_error_fmt(hdl, EZFS_BADCACHE, - dgettext(TEXT_DOMAIN, "blkid_dev_set_search() %d"), err); - goto err_blkid3; + error = blkid_dev_set_search(iter, "TYPE", "zfs_member"); + if (error != 0) { + blkid_dev_iterate_end(iter); + blkid_put_cache(cache); + return (error); } - while (blkid_dev_next(iter, &dev) == 0) { - nvlist_t *label; - char *devname; - int fd, num_labels; - - devname = (char *) blkid_dev_devname(dev); - if ((fd = open64(devname, O_RDONLY)) < 0) - continue; - - err = zpool_read_label(fd, &label, &num_labels); - (void) close(fd); + *slice_cache = zfs_alloc(hdl, sizeof (avl_tree_t)); + avl_create(*slice_cache, slice_cache_compare, sizeof (rdsk_node_t), + offsetof(rdsk_node_t, rn_node)); - if (err || label == NULL) - continue; - - add_configs_from_label(hdl, pools, devname, num_labels, label); + while (blkid_dev_next(iter, &dev) == 0) { + slice = zfs_alloc(hdl, sizeof (rdsk_node_t)); + slice->rn_name = zfs_strdup(hdl, blkid_dev_devname(dev)); + slice->rn_lock = lock; + slice->rn_avl = *slice_cache; + slice->rn_hdl = hdl; + slice->rn_order = 100; + slice->rn_labelpaths = B_TRUE; + mutex_enter(lock); + avl_add(*slice_cache, slice); + mutex_exit(lock); } - err = 0; -err_blkid3: blkid_dev_iterate_end(iter); -err_blkid2: blkid_put_cache(cache); -err_blkid1: - return (err); + + return (0); } char * @@ -1797,176 +1789,98 @@ zpool_default_import_path[DEFAULT_IMPORT_PATH_SIZE] = { static nvlist_t * zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg) { - int i, dirs = iarg->paths; - struct dirent64 *dp; - char path[MAXPATHLEN]; - char *end, **dir = iarg->path; - size_t pathleft; nvlist_t *ret = NULL; pool_list_t pools = { 0 }; pool_entry_t *pe, *penext; vdev_entry_t *ve, *venext; config_entry_t *ce, *cenext; name_entry_t *ne, *nenext; - avl_tree_t slice_cache; + kmutex_t lock; + avl_tree_t *cache; rdsk_node_t *slice; void *cookie; + taskq_t *t; verify(iarg->poolname == NULL || iarg->guid == 0); + mutex_init(&lock, NULL, MUTEX_DEFAULT, NULL); /* - * Prefer to locate pool member vdevs using libblkid. Only fall - * back to legacy directory scanning when explicitly requested or - * if an error is encountered when consulted the libblkid cache. + * Locate pool member vdevs using libblkid or by directory scanning. + * On success a newly allocated AVL tree which is populated with an + * entry for each discovered vdev will be returned as the cache. + * It's the callers responsibility to consume and destroy this tree. */ - if (dirs == 0) { - if (!iarg->scan && (zpool_find_import_blkid(hdl, &pools) == 0)) - goto skip_scanning; + if (iarg->scan || iarg->paths != 0) { + int dirs = iarg->paths; + char **dir = iarg->path; - dir = zpool_default_import_path; - dirs = DEFAULT_IMPORT_PATH_SIZE; + if (dirs == 0) { + dir = zpool_default_import_path; + dirs = DEFAULT_IMPORT_PATH_SIZE; + } + + if (zpool_find_import_scan(hdl, &lock, &cache, dir, dirs) != 0) + return (NULL); + } else { + if (zpool_find_import_blkid(hdl, &lock, &cache) != 0) + return (NULL); } /* - * Go through and read the label configuration information from every - * possible device, organizing the information according to pool GUID - * and toplevel GUID. + * Create a thread pool to parallelize the process of reading and + * validating labels, a large number of threads can be used due to + * minimal contention. */ - for (i = 0; i < dirs; i++) { - taskq_t *t; - char *rdsk; - int dfd; - boolean_t config_failed = B_FALSE; - DIR *dirp; - - /* use realpath to normalize the path */ - if (realpath(dir[i], path) == 0) { + t = taskq_create("z_import", 2 * boot_ncpus, defclsyspri, + 2 * boot_ncpus, INT_MAX, TASKQ_PREPOPULATE); - /* it is safe to skip missing search paths */ - if (errno == ENOENT) - continue; + for (slice = avl_first(cache); slice; + (slice = avl_walk(cache, slice, AVL_AFTER))) + (void) taskq_dispatch(t, zpool_open_func, slice, TQ_SLEEP); - zfs_error_aux(hdl, strerror(errno)); - (void) zfs_error_fmt(hdl, EZFS_BADPATH, - dgettext(TEXT_DOMAIN, "cannot open '%s'"), dir[i]); - goto error; - } - end = &path[strlen(path)]; - *end++ = '/'; - *end = 0; - pathleft = &path[sizeof (path)] - end; + taskq_wait(t); + taskq_destroy(t); - /* - * Using raw devices instead of block devices when we're - * reading the labels skips a bunch of slow operations during - * close(2) processing, so we replace /dev/dsk with /dev/rdsk. - */ - if (strcmp(path, "/dev/dsk/") == 0) - rdsk = "/dev/rdsk/"; - else - rdsk = path; - - if ((dfd = open64(rdsk, O_RDONLY)) < 0 || - (dirp = fdopendir(dfd)) == NULL) { - if (dfd >= 0) - (void) close(dfd); - zfs_error_aux(hdl, strerror(errno)); - (void) zfs_error_fmt(hdl, EZFS_BADPATH, - dgettext(TEXT_DOMAIN, "cannot open '%s'"), - rdsk); - goto error; - } - - avl_create(&slice_cache, slice_cache_compare, - sizeof (rdsk_node_t), offsetof(rdsk_node_t, rn_node)); - - /* - * This is not MT-safe, but we have no MT consumers of libzfs - */ - while ((dp = readdir64(dirp)) != NULL) { - const char *name = dp->d_name; - if (name[0] == '.' && - (name[1] == 0 || (name[1] == '.' && name[2] == 0))) - continue; - - slice = zfs_alloc(hdl, sizeof (rdsk_node_t)); - slice->rn_name = zfs_strdup(hdl, name); - slice->rn_avl = &slice_cache; - slice->rn_dfd = dfd; - slice->rn_hdl = hdl; - slice->rn_nozpool = B_FALSE; - avl_add(&slice_cache, slice); - } - - /* - * create a thread pool to do all of this in parallel; - * rn_nozpool is not protected, so this is racy in that - * multiple tasks could decide that the same slice can - * not hold a zpool, which is benign. Also choose - * double the number of processors; we hold a lot of - * locks in the kernel, so going beyond this doesn't - * buy us much. - */ - t = taskq_create("z_import", 2 * boot_ncpus, defclsyspri, - 2 * boot_ncpus, INT_MAX, TASKQ_PREPOPULATE); - for (slice = avl_first(&slice_cache); slice; - (slice = avl_walk(&slice_cache, slice, - AVL_AFTER))) - (void) taskq_dispatch(t, zpool_open_func, slice, - TQ_SLEEP); - taskq_wait(t); - taskq_destroy(t); - - cookie = NULL; - while ((slice = avl_destroy_nodes(&slice_cache, - &cookie)) != NULL) { - if (slice->rn_config != NULL && !config_failed) { - nvlist_t *config = slice->rn_config; - boolean_t matched = B_TRUE; - - if (iarg->poolname != NULL) { - char *pname; - - matched = nvlist_lookup_string(config, - ZPOOL_CONFIG_POOL_NAME, - &pname) == 0 && - strcmp(iarg->poolname, pname) == 0; - } else if (iarg->guid != 0) { - uint64_t this_guid; - - matched = nvlist_lookup_uint64(config, - ZPOOL_CONFIG_POOL_GUID, - &this_guid) == 0 && - iarg->guid == this_guid; - } - if (!matched) { - nvlist_free(config); - } else { - /* - * use the non-raw path for the config - */ - (void) strlcpy(end, slice->rn_name, - pathleft); - if (add_config(hdl, &pools, path, i+1, - slice->rn_num_labels, config) != 0) - config_failed = B_TRUE; - } + /* + * Process the cache filtering out any entries which are not + * for the specificed pool then adding matching label configs. + */ + cookie = NULL; + while ((slice = avl_destroy_nodes(cache, &cookie)) != NULL) { + if (slice->rn_config != NULL) { + nvlist_t *config = slice->rn_config; + boolean_t matched = B_TRUE; + + if (iarg->poolname != NULL) { + char *pname; + + matched = nvlist_lookup_string(config, + ZPOOL_CONFIG_POOL_NAME, &pname) == 0 && + strcmp(iarg->poolname, pname) == 0; + } else if (iarg->guid != 0) { + uint64_t this_guid; + + matched = nvlist_lookup_uint64(config, + ZPOOL_CONFIG_POOL_GUID, &this_guid) == 0 && + iarg->guid == this_guid; + } + if (!matched) { + nvlist_free(config); + } else { + add_config(hdl, &pools, + slice->rn_name, slice->rn_order, + slice->rn_num_labels, config); } - free(slice->rn_name); - free(slice); } - avl_destroy(&slice_cache); - - (void) closedir(dirp); - - if (config_failed) - goto error; + free(slice->rn_name); + free(slice); } + avl_destroy(cache); + free(cache); + mutex_destroy(&lock); -skip_scanning: ret = get_configs(hdl, &pools, iarg->can_be_active); -error: for (pe = pools.pools; pe != NULL; pe = penext) { penext = pe->pe_next; for (ve = pe->pe_vdevs; ve != NULL; ve = venext) { diff --git a/lib/libzfs/libzfs_iter.c b/lib/libzfs/libzfs_iter.c index 5c1cf966a543..96f6574651de 100644 --- a/lib/libzfs/libzfs_iter.c +++ b/lib/libzfs/libzfs_iter.c @@ -197,7 +197,7 @@ zfs_iter_bookmarks(zfs_handle_t *zhp, zfs_iter_f func, void *data) for (pair = nvlist_next_nvpair(bmarks, NULL); pair != NULL; pair = nvlist_next_nvpair(bmarks, pair)) { - char name[ZFS_MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; char *bmark_name; nvlist_t *bmark_props; @@ -384,7 +384,7 @@ zfs_iter_snapspec(zfs_handle_t *fs_zhp, const char *spec_orig, * exists. */ if (ssa.ssa_last[0] != '\0') { - char snapname[ZFS_MAXNAMELEN]; + char snapname[ZFS_MAX_DATASET_NAME_LEN]; (void) snprintf(snapname, sizeof (snapname), "%s@%s", zfs_get_name(fs_zhp), ssa.ssa_last); @@ -404,7 +404,7 @@ zfs_iter_snapspec(zfs_handle_t *fs_zhp, const char *spec_orig, ret = ENOENT; } } else { - char snapname[ZFS_MAXNAMELEN]; + char snapname[ZFS_MAX_DATASET_NAME_LEN]; zfs_handle_t *snap_zhp; (void) snprintf(snapname, sizeof (snapname), "%s@%s", zfs_get_name(fs_zhp), comma_separated); diff --git a/lib/libzfs/libzfs_mount.c b/lib/libzfs/libzfs_mount.c index 29907dc8bfac..5e57adab0c9d 100644 --- a/lib/libzfs/libzfs_mount.c +++ b/lib/libzfs/libzfs_mount.c @@ -230,7 +230,7 @@ static boolean_t zfs_is_mountable(zfs_handle_t *zhp, char *buf, size_t buflen, zprop_source_t *source) { - char sourceloc[ZFS_MAXNAMELEN]; + char sourceloc[MAXNAMELEN]; zprop_source_t sourcetype; if (!zfs_prop_valid_for_type(ZFS_PROP_MOUNTPOINT, zhp->zfs_type, @@ -1051,6 +1051,17 @@ mount_cb(zfs_handle_t *zhp, void *data) return (0); } + /* + * If this filesystem is inconsistent and has a receive resume + * token, we can not mount it. + */ + if (zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT) && + zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN, + NULL, 0, NULL, NULL, 0, B_TRUE) == 0) { + zfs_close(zhp); + return (0); + } + libzfs_add_handle(cbp, zhp); if (zfs_iter_filesystems(zhp, mount_cb, cbp) != 0) { zfs_close(zhp); diff --git a/lib/libzfs/libzfs_pool.c b/lib/libzfs/libzfs_pool.c index 8b74e5da6678..2484ddc12a58 100644 --- a/lib/libzfs/libzfs_pool.c +++ b/lib/libzfs/libzfs_pool.c @@ -441,7 +441,7 @@ pool_uses_efi(nvlist_t *config) boolean_t zpool_is_bootable(zpool_handle_t *zhp) { - char bootfs[ZPOOL_MAXNAMELEN]; + char bootfs[ZFS_MAX_DATASET_NAME_LEN]; return (zpool_get_prop(zhp, ZPOOL_PROP_BOOTFS, bootfs, sizeof (bootfs), NULL, B_FALSE) == 0 && strncmp(bootfs, "-", @@ -1907,7 +1907,12 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname, "one or more devices are already in use\n")); (void) zfs_error(hdl, EZFS_BADDEV, desc); break; - + case ENAMETOOLONG: + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "new name of at least one dataset is longer than " + "the maximum allowable length")); + (void) zfs_error(hdl, EZFS_NAMETOOLONG, desc); + break; default: (void) zpool_standard_error(hdl, error, desc); zpool_explain_recover(hdl, @@ -4006,7 +4011,7 @@ zpool_obj_to_path(zpool_handle_t *zhp, uint64_t dsobj, uint64_t obj, zfs_cmd_t zc = {"\0"}; boolean_t mounted = B_FALSE; char *mntpnt = NULL; - char dsname[MAXNAMELEN]; + char dsname[ZFS_MAX_DATASET_NAME_LEN]; if (dsobj == 0) { /* special case for the MOS */ diff --git a/lib/libzfs/libzfs_sendrecv.c b/lib/libzfs/libzfs_sendrecv.c index 46640f623243..bc8bd5f8a8bc 100644 --- a/lib/libzfs/libzfs_sendrecv.c +++ b/lib/libzfs/libzfs_sendrecv.c @@ -21,11 +21,12 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2014 by Delphix. All rights reserved. + * Copyright (c) 2011, 2015 by Delphix. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. * Copyright (c) 2012 Pawel Jakub Dawidek . * All rights reserved * Copyright (c) 2013 Steven Hartland. All rights reserved. + * Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved. */ #include @@ -56,6 +57,7 @@ #include "zfs_prop.h" #include "zfs_fletcher.h" #include "libzfs_impl.h" +#include #include #include #include @@ -65,7 +67,9 @@ extern void zfs_setprop_error(libzfs_handle_t *, zfs_prop_t, int, char *); static int zfs_receive_impl(libzfs_handle_t *, const char *, const char *, recvflags_t *, int, const char *, nvlist_t *, avl_tree_t *, char **, int, - uint64_t *); + uint64_t *, const char *); +static int guid_to_name(libzfs_handle_t *, const char *, + uint64_t, boolean_t, char *); static const zio_cksum_t zero_cksum = { { 0 } }; @@ -234,7 +238,7 @@ cksummer(void *arg) { dedup_arg_t *dda = arg; char *buf = zfs_alloc(dda->dedup_hdl, SPA_MAXBLOCKSIZE); - dmu_replay_record_t thedrr; + dmu_replay_record_t thedrr = { 0 }; dmu_replay_record_t *drr = &thedrr; FILE *ofp; int outfd; @@ -283,8 +287,7 @@ cksummer(void *arg) DMU_BACKUP_FEATURE_DEDUPPROPS); DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, fflags); - if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == - DMU_COMPOUNDSTREAM && drr->drr_payloadlen != 0) { + if (drr->drr_payloadlen != 0) { sz = drr->drr_payloadlen; if (sz > SPA_MAXBLOCKSIZE) { @@ -834,7 +837,7 @@ typedef struct send_dump_data { /* these are all just the short snapname (the part after the @) */ const char *fromsnap; const char *tosnap; - char prevsnap[ZFS_MAXNAMELEN]; + char prevsnap[ZFS_MAX_DATASET_NAME_LEN]; uint64_t prevsnap_obj; boolean_t seenfrom, seento, replicate, doall, fromorigin; boolean_t verbose, dryrun, parsable, progress, embed_data, std_out; @@ -847,7 +850,7 @@ typedef struct send_dump_data { snapfilter_cb_t *filter_cb; void *filter_cb_arg; nvlist_t *debugnv; - char holdtag[ZFS_MAXNAMELEN]; + char holdtag[ZFS_MAX_DATASET_NAME_LEN]; int cleanup_fd; uint64_t size; } send_dump_data_t; @@ -1013,17 +1016,14 @@ static void * send_progress_thread(void *arg) { progress_arg_t *pa = arg; - zfs_cmd_t zc = {"\0"}; zfs_handle_t *zhp = pa->pa_zhp; libzfs_handle_t *hdl = zhp->zfs_hdl; unsigned long long bytes; char buf[16]; - time_t t; struct tm *tm; - assert(zhp->zfs_type == ZFS_TYPE_SNAPSHOT); (void) strlcpy(zc.zc_name, zhp->zfs_name, sizeof (zc.zc_name)); if (!pa->pa_parsable) @@ -1056,6 +1056,51 @@ send_progress_thread(void *arg) } } +static void +send_print_verbose(FILE *fout, const char *tosnap, const char *fromsnap, + uint64_t size, boolean_t parsable) +{ + if (parsable) { + if (fromsnap != NULL) { + (void) fprintf(fout, "incremental\t%s\t%s", + fromsnap, tosnap); + } else { + (void) fprintf(fout, "full\t%s", + tosnap); + } + } else { + if (fromsnap != NULL) { + if (strchr(fromsnap, '@') == NULL && + strchr(fromsnap, '#') == NULL) { + (void) fprintf(fout, dgettext(TEXT_DOMAIN, + "send from @%s to %s"), + fromsnap, tosnap); + } else { + (void) fprintf(fout, dgettext(TEXT_DOMAIN, + "send from %s to %s"), + fromsnap, tosnap); + } + } else { + (void) fprintf(fout, dgettext(TEXT_DOMAIN, + "full send of %s"), + tosnap); + } + } + + if (size != 0) { + if (parsable) { + (void) fprintf(fout, "\t%llu", + (longlong_t)size); + } else { + char buf[16]; + zfs_nicenum(size, buf, sizeof (buf)); + (void) fprintf(fout, dgettext(TEXT_DOMAIN, + " estimated size is %s"), buf); + } + } + (void) fprintf(fout, "\n"); +} + static int dump_snapshot(zfs_handle_t *zhp, void *arg) { @@ -1135,37 +1180,14 @@ dump_snapshot(zfs_handle_t *zhp, void *arg) (sdd->fromorigin || sdd->replicate); if (sdd->verbose) { - uint64_t size; - err = estimate_ioctl(zhp, sdd->prevsnap_obj, + uint64_t size = 0; + (void) estimate_ioctl(zhp, sdd->prevsnap_obj, fromorigin, &size); - if (sdd->parsable) { - if (sdd->prevsnap[0] != '\0') { - (void) fprintf(fout, "incremental\t%s\t%s", - sdd->prevsnap, zhp->zfs_name); - } else { - (void) fprintf(fout, "full\t%s", - zhp->zfs_name); - } - } else { - (void) fprintf(fout, dgettext(TEXT_DOMAIN, - "send from @%s to %s"), - sdd->prevsnap, zhp->zfs_name); - } - if (err == 0) { - if (sdd->parsable) { - (void) fprintf(fout, "\t%llu\n", - (longlong_t)size); - } else { - char buf[16]; - zfs_nicenum(size, buf, sizeof (buf)); - (void) fprintf(fout, dgettext(TEXT_DOMAIN, - " estimated size is %s\n"), buf); - } - sdd->size += size; - } else { - (void) fprintf(fout, "\n"); - } + send_print_verbose(fout, zhp->zfs_name, + sdd->prevsnap[0] ? sdd->prevsnap : NULL, + size, sdd->parsable); + sdd->size += size; } if (!sdd->dryrun) { @@ -1376,6 +1398,231 @@ dump_filesystems(zfs_handle_t *rzhp, void *arg) return (0); } +nvlist_t * +zfs_send_resume_token_to_nvlist(libzfs_handle_t *hdl, const char *token) +{ + unsigned int version; + int nread, i; + unsigned long long checksum, packed_len; + + /* + * Decode token header, which is: + * -- + * Note that the only supported token version is 1. + */ + nread = sscanf(token, "%u-%llx-%llx-", + &version, &checksum, &packed_len); + if (nread != 3) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "resume token is corrupt (invalid format)")); + return (NULL); + } + + if (version != ZFS_SEND_RESUME_TOKEN_VERSION) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "resume token is corrupt (invalid version %u)"), + version); + return (NULL); + } + + /* convert hexadecimal representation to binary */ + token = strrchr(token, '-') + 1; + int len = strlen(token) / 2; + unsigned char *compressed = zfs_alloc(hdl, len); + for (i = 0; i < len; i++) { + nread = sscanf(token + i * 2, "%2hhx", compressed + i); + if (nread != 1) { + free(compressed); + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "resume token is corrupt " + "(payload is not hex-encoded)")); + return (NULL); + } + } + + /* verify checksum */ + zio_cksum_t cksum; + fletcher_4_native(compressed, len, &cksum); + if (cksum.zc_word[0] != checksum) { + free(compressed); + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "resume token is corrupt (incorrect checksum)")); + return (NULL); + } + + /* uncompress */ + void *packed = zfs_alloc(hdl, packed_len); + uLongf packed_len_long = packed_len; + if (uncompress(packed, &packed_len_long, compressed, len) != Z_OK || + packed_len_long != packed_len) { + free(packed); + free(compressed); + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "resume token is corrupt (decompression failed)")); + return (NULL); + } + + /* unpack nvlist */ + nvlist_t *nv; + int error = nvlist_unpack(packed, packed_len, &nv, KM_SLEEP); + free(packed); + free(compressed); + if (error != 0) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "resume token is corrupt (nvlist_unpack failed)")); + return (NULL); + } + return (nv); +} + +int +zfs_send_resume(libzfs_handle_t *hdl, sendflags_t *flags, int outfd, + const char *resume_token) +{ + char errbuf[1024]; + char *toname; + char *fromname = NULL; + uint64_t resumeobj, resumeoff, toguid, fromguid, bytes; + zfs_handle_t *zhp; + int error = 0; + char name[ZFS_MAX_DATASET_NAME_LEN]; + enum lzc_send_flags lzc_flags = 0; + + (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, + "cannot resume send")); + + nvlist_t *resume_nvl = + zfs_send_resume_token_to_nvlist(hdl, resume_token); + if (resume_nvl == NULL) { + /* + * zfs_error_aux has already been set by + * zfs_send_resume_token_to_nvlist + */ + return (zfs_error(hdl, EZFS_FAULT, errbuf)); + } + if (flags->verbose) { + (void) fprintf(stderr, dgettext(TEXT_DOMAIN, + "resume token contents:\n")); + nvlist_print(stderr, resume_nvl); + } + + if (nvlist_lookup_string(resume_nvl, "toname", &toname) != 0 || + nvlist_lookup_uint64(resume_nvl, "object", &resumeobj) != 0 || + nvlist_lookup_uint64(resume_nvl, "offset", &resumeoff) != 0 || + nvlist_lookup_uint64(resume_nvl, "bytes", &bytes) != 0 || + nvlist_lookup_uint64(resume_nvl, "toguid", &toguid) != 0) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "resume token is corrupt")); + return (zfs_error(hdl, EZFS_FAULT, errbuf)); + } + fromguid = 0; + (void) nvlist_lookup_uint64(resume_nvl, "fromguid", &fromguid); + + if (flags->embed_data || nvlist_exists(resume_nvl, "embedok")) + lzc_flags |= LZC_SEND_FLAG_EMBED_DATA; + + if (guid_to_name(hdl, toname, toguid, B_FALSE, name) != 0) { + if (zfs_dataset_exists(hdl, toname, ZFS_TYPE_DATASET)) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "'%s' is no longer the same snapshot used in " + "the initial send"), toname); + } else { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "'%s' used in the initial send no longer exists"), + toname); + } + return (zfs_error(hdl, EZFS_BADPATH, errbuf)); + } + zhp = zfs_open(hdl, name, ZFS_TYPE_DATASET); + if (zhp == NULL) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "unable to access '%s'"), name); + return (zfs_error(hdl, EZFS_BADPATH, errbuf)); + } + + if (fromguid != 0) { + if (guid_to_name(hdl, toname, fromguid, B_TRUE, name) != 0) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "incremental source %#llx no longer exists"), + (longlong_t)fromguid); + return (zfs_error(hdl, EZFS_BADPATH, errbuf)); + } + fromname = name; + } + + if (flags->verbose) { + uint64_t size = 0; + error = lzc_send_space(zhp->zfs_name, fromname, &size); + if (error == 0) + size = MAX(0, (int64_t)(size - bytes)); + send_print_verbose(stderr, zhp->zfs_name, fromname, + size, flags->parsable); + } + + if (!flags->dryrun) { + progress_arg_t pa = { 0 }; + pthread_t tid; + /* + * If progress reporting is requested, spawn a new thread to + * poll ZFS_IOC_SEND_PROGRESS at a regular interval. + */ + if (flags->progress) { + pa.pa_zhp = zhp; + pa.pa_fd = outfd; + pa.pa_parsable = flags->parsable; + + error = pthread_create(&tid, NULL, + send_progress_thread, &pa); + if (error != 0) { + zfs_close(zhp); + return (error); + } + } + + error = lzc_send_resume(zhp->zfs_name, fromname, outfd, + lzc_flags, resumeobj, resumeoff); + + if (flags->progress) { + (void) pthread_cancel(tid); + (void) pthread_join(tid, NULL); + } + + char errbuf[1024]; + (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, + "warning: cannot send '%s'"), zhp->zfs_name); + + zfs_close(zhp); + + switch (error) { + case 0: + return (0); + case EXDEV: + case ENOENT: + case EDQUOT: + case EFBIG: + case EIO: + case ENOLINK: + case ENOSPC: + case ENOSTR: + case ENXIO: + case EPIPE: + case ERANGE: + case EFAULT: + case EROFS: + zfs_error_aux(hdl, strerror(errno)); + return (zfs_error(hdl, EZFS_BADBACKUP, errbuf)); + + default: + return (zfs_standard_error(hdl, errno, errbuf)); + } + } + + + zfs_close(zhp); + + return (error); +} + /* * Generate a send stream for the dataset identified by the argument zhp. * @@ -1451,7 +1698,9 @@ zfs_send(zfs_handle_t *zhp, const char *fromsnap, const char *tosnap, dmu_replay_record_t drr = { 0 }; char *packbuf = NULL; size_t buflen = 0; - zio_cksum_t zc = { { 0 } }; + zio_cksum_t zc; + + ZIO_SET_CHECKSUM(&zc, 0, 0, 0, 0); if (flags->replicate || flags->props) { nvlist_t *hdrnv; @@ -1827,8 +2076,8 @@ recv_rename(libzfs_handle_t *hdl, const char *name, const char *tryname, if (err != 0 && strncmp(name + baselen, "recv-", 5) != 0) { seq++; - (void) snprintf(newname, ZFS_MAXNAMELEN, "%.*srecv-%u-%u", - baselen, name, getpid(), seq); + (void) snprintf(newname, ZFS_MAX_DATASET_NAME_LEN, + "%.*srecv-%u-%u", baselen, name, getpid(), seq); (void) strlcpy(zc.zc_value, newname, sizeof (zc.zc_value)); if (flags->verbose) { @@ -1913,6 +2162,7 @@ recv_destroy(libzfs_handle_t *hdl, const char *name, int baselen, typedef struct guid_to_name_data { uint64_t guid; + boolean_t bookmark_ok; char *name; char *skip; } guid_to_name_data_t; @@ -1921,20 +2171,25 @@ static int guid_to_name_cb(zfs_handle_t *zhp, void *arg) { guid_to_name_data_t *gtnd = arg; + const char *slash; int err; if (gtnd->skip != NULL && - strcmp(zhp->zfs_name, gtnd->skip) == 0) { + (slash = strrchr(zhp->zfs_name, '/')) != NULL && + strcmp(slash + 1, gtnd->skip) == 0) { + zfs_close(zhp); return (0); } - if (zhp->zfs_dmustats.dds_guid == gtnd->guid) { + if (zfs_prop_get_int(zhp, ZFS_PROP_GUID) == gtnd->guid) { (void) strcpy(gtnd->name, zhp->zfs_name); zfs_close(zhp); return (EEXIST); } err = zfs_iter_children(zhp, guid_to_name_cb, gtnd); + if (err != EEXIST && gtnd->bookmark_ok) + err = zfs_iter_bookmarks(zhp, guid_to_name_cb, gtnd); zfs_close(zhp); return (err); } @@ -1948,45 +2203,48 @@ guid_to_name_cb(zfs_handle_t *zhp, void *arg) */ static int guid_to_name(libzfs_handle_t *hdl, const char *parent, uint64_t guid, - char *name) + boolean_t bookmark_ok, char *name) { - /* exhaustive search all local snapshots */ - char pname[ZFS_MAXNAMELEN]; + char pname[ZFS_MAX_DATASET_NAME_LEN]; guid_to_name_data_t gtnd; - int err = 0; - zfs_handle_t *zhp; - char *cp; gtnd.guid = guid; + gtnd.bookmark_ok = bookmark_ok; gtnd.name = name; gtnd.skip = NULL; - (void) strlcpy(pname, parent, sizeof (pname)); - /* - * Search progressively larger portions of the hierarchy. This will + * Search progressively larger portions of the hierarchy, starting + * with the filesystem specified by 'parent'. This will * select the "most local" version of the origin snapshot in the case * that there are multiple matching snapshots in the system. */ - while ((cp = strrchr(pname, '/')) != NULL) { - + (void) strlcpy(pname, parent, sizeof (pname)); + char *cp = strrchr(pname, '@'); + if (cp == NULL) + cp = strchr(pname, '\0'); + for (; cp != NULL; cp = strrchr(pname, '/')) { /* Chop off the last component and open the parent */ *cp = '\0'; - zhp = make_dataset_handle(hdl, pname); + zfs_handle_t *zhp = make_dataset_handle(hdl, pname); if (zhp == NULL) continue; - - err = zfs_iter_children(zhp, guid_to_name_cb, >nd); + int err = guid_to_name_cb(zfs_handle_dup(zhp), >nd); + if (err != EEXIST) + err = zfs_iter_children(zhp, guid_to_name_cb, >nd); + if (err != EEXIST && bookmark_ok) + err = zfs_iter_bookmarks(zhp, guid_to_name_cb, >nd); zfs_close(zhp); if (err == EEXIST) return (0); /* - * Remember the dataset that we already searched, so we - * skip it next time through. + * Remember the last portion of the dataset so we skip it next + * time through (as we've already searched that portion of the + * hierarchy). */ - gtnd.skip = pname; + gtnd.skip = strrchr(pname, '/') + 1; } return (ENOENT); @@ -2002,7 +2260,7 @@ created_before(libzfs_handle_t *hdl, avl_tree_t *avl, { nvlist_t *nvfs; char *fsname = NULL, *snapname = NULL; - char buf[ZFS_MAXNAMELEN]; + char buf[ZFS_MAX_DATASET_NAME_LEN]; int rv; zfs_handle_t *guid1hdl, *guid2hdl; uint64_t create1, create2; @@ -2053,7 +2311,7 @@ recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs, avl_tree_t *local_avl; nvpair_t *fselem, *nextfselem; char *fromsnap; - char newname[ZFS_MAXNAMELEN]; + char newname[ZFS_MAX_DATASET_NAME_LEN]; char guidname[32]; int error; boolean_t needagain, progress, recursive; @@ -2172,7 +2430,7 @@ recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs, /* check for delete */ if (found == NULL) { - char name[ZFS_MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; if (!flags->force) continue; @@ -2213,8 +2471,8 @@ recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs, /* check for different snapname */ if (strcmp(nvpair_name(snapelem), stream_snapname) != 0) { - char name[ZFS_MAXNAMELEN]; - char tryname[ZFS_MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; + char tryname[ZFS_MAX_DATASET_NAME_LEN]; (void) snprintf(name, sizeof (name), "%s@%s", fsname, nvpair_name(snapelem)); @@ -2298,7 +2556,7 @@ recv_incremental_replication(libzfs_handle_t *hdl, const char *tofs, ((flags->isprefix || strcmp(tofs, fsname) != 0) && (s1 != NULL) && (s2 != NULL) && strcmp(s1, s2) != 0)) { nvlist_t *parent; - char tryname[ZFS_MAXNAMELEN]; + char tryname[ZFS_MAX_DATASET_NAME_LEN]; parent = fsavl_find(local_avl, stream_parent_fromsnap_guid, NULL); @@ -2364,9 +2622,10 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, nvlist_t *stream_nv = NULL; avl_tree_t *stream_avl = NULL; char *fromsnap = NULL; + char *sendsnap = NULL; char *cp; - char tofs[ZFS_MAXNAMELEN]; - char sendfs[ZFS_MAXNAMELEN]; + char tofs[ZFS_MAX_DATASET_NAME_LEN]; + char sendfs[ZFS_MAX_DATASET_NAME_LEN]; char errbuf[1024]; dmu_replay_record_t drre; int error; @@ -2450,7 +2709,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, nvlist_t *renamed = NULL; nvpair_t *pair = NULL; - (void) strlcpy(tofs, destname, ZFS_MAXNAMELEN); + (void) strlcpy(tofs, destname, sizeof (tofs)); if (flags->isprefix) { struct drr_begin *drrb = &drr->drr_u.drr_begin; int i; @@ -2459,7 +2718,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, cp = strrchr(drrb->drr_toname, '/'); if (cp == NULL) { (void) strlcat(tofs, "/", - ZFS_MAXNAMELEN); + sizeof (tofs)); i = 0; } else { i = (cp - drrb->drr_toname); @@ -2469,7 +2728,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, } /* zfs_receive_one() will create_parents() */ (void) strlcat(tofs, &drrb->drr_toname[i], - ZFS_MAXNAMELEN); + sizeof (tofs)); *strchr(tofs, '@') = '\0'; } @@ -2511,9 +2770,17 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, * zfs_receive_one(). */ (void) strlcpy(sendfs, drr->drr_u.drr_begin.drr_toname, - ZFS_MAXNAMELEN); - if ((cp = strchr(sendfs, '@')) != NULL) + sizeof (sendfs)); + if ((cp = strchr(sendfs, '@')) != NULL) { *cp = '\0'; + /* + * Find the "sendsnap", the final snapshot in a replication + * stream. zfs_receive_one() handles certain errors + * differently, depending on if the contained stream is the + * last one or not. + */ + sendsnap = (cp + 1); + } /* Finally, receive each contained stream */ do { @@ -2526,7 +2793,7 @@ zfs_receive_package(libzfs_handle_t *hdl, int fd, const char *destname, */ error = zfs_receive_impl(hdl, destname, NULL, flags, fd, sendfs, stream_nv, stream_avl, top_zfs, cleanup_fd, - action_handlep); + action_handlep, sendsnap); if (error == ENODATA) { error = 0; break; @@ -2586,11 +2853,9 @@ recv_skip(libzfs_handle_t *hdl, int fd, boolean_t byteswap) switch (drr->drr_type) { case DRR_BEGIN: - /* NB: not to be used on v2 stream packages */ if (drr->drr_payloadlen != 0) { - zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "invalid substream header")); - return (zfs_error(hdl, EZFS_BADSTREAM, errbuf)); + (void) recv_read(hdl, fd, buf, + drr->drr_payloadlen, B_FALSE, NULL); } break; @@ -2651,6 +2916,40 @@ recv_skip(libzfs_handle_t *hdl, int fd, boolean_t byteswap) return (-1); } +static void +recv_ecksum_set_aux(libzfs_handle_t *hdl, const char *target_snap, + boolean_t resumable) +{ + char target_fs[ZFS_MAX_DATASET_NAME_LEN]; + + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "checksum mismatch or incomplete stream")); + + if (!resumable) + return; + (void) strlcpy(target_fs, target_snap, sizeof (target_fs)); + *strchr(target_fs, '@') = '\0'; + zfs_handle_t *zhp = zfs_open(hdl, target_fs, + ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME); + if (zhp == NULL) + return; + + char token_buf[ZFS_MAXPROPLEN]; + int error = zfs_prop_get(zhp, ZFS_PROP_RECEIVE_RESUME_TOKEN, + token_buf, sizeof (token_buf), + NULL, NULL, 0, B_TRUE); + if (error == 0) { + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "checksum mismatch or incomplete stream.\n" + "Partially received snapshot is saved.\n" + "A resuming stream can be generated on the sending " + "system by running:\n" + " zfs send -t %s"), + token_buf); + } + zfs_close(zhp); +} + /* * Restores a backup of tosnap from the file descriptor specified by infd. */ @@ -2659,25 +2958,33 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, const char *originsnap, recvflags_t *flags, dmu_replay_record_t *drr, dmu_replay_record_t *drr_noswap, const char *sendfs, nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd, - uint64_t *action_handlep) + uint64_t *action_handlep, const char *finalsnap) { - zfs_cmd_t zc = {"\0"}; time_t begin_time; int ioctl_err, ioctl_errno, err; char *cp; struct drr_begin *drrb = &drr->drr_u.drr_begin; char errbuf[1024]; - char prop_errbuf[1024]; const char *chopprefix; boolean_t newfs = B_FALSE; boolean_t stream_wantsnewfs; + boolean_t newprops = B_FALSE; + uint64_t read_bytes = 0; + uint64_t errflags = 0; uint64_t parent_snapguid = 0; prop_changelist_t *clp = NULL; nvlist_t *snapprops_nvlist = NULL; zprop_errflags_t prop_errflags; + nvlist_t *prop_errors = NULL; boolean_t recursive; + char *snapname = NULL; + char destsnap[MAXPATHLEN * 2]; + char origin[MAXNAMELEN]; + char name[MAXPATHLEN]; + nvlist_t *props = NULL; begin_time = time(NULL); + bzero(origin, MAXNAMELEN); (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, "cannot receive")); @@ -2686,27 +2993,21 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, ENOENT); if (stream_avl != NULL) { - char *snapname; nvlist_t *fs = fsavl_find(stream_avl, drrb->drr_toguid, &snapname); - nvlist_t *props; - int ret; (void) nvlist_lookup_uint64(fs, "parentfromsnap", &parent_snapguid); err = nvlist_lookup_nvlist(fs, "props", &props); - if (err) + if (err) { VERIFY(0 == nvlist_alloc(&props, NV_UNIQUE_NAME, 0)); + newprops = B_TRUE; + } if (flags->canmountoff) { VERIFY(0 == nvlist_add_uint64(props, zfs_prop_to_name(ZFS_PROP_CANMOUNT), 0)); } - ret = zcmd_write_src_nvlist(hdl, &zc, props); - if (err) - nvlist_free(props); - if (ret != 0) - return (-1); } cp = NULL; @@ -2727,7 +3028,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, if (strchr(tosnap, '@')) { zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid " "argument - snapshot not allowed with -e")); - return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf)); + err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf); + goto out; } chopprefix = strrchr(sendfs, '/'); @@ -2754,7 +3056,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, if (strchr(tosnap, '@')) { zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "invalid " "argument - snapshot not allowed with -d")); - return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf)); + err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf); + goto out; } chopprefix = strchr(drrb->drr_toname, '/'); @@ -2772,7 +3075,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "cannot specify snapshot name for multi-snapshot " "stream")); - return (zfs_error(hdl, EZFS_BADSTREAM, errbuf)); + err = zfs_error(hdl, EZFS_BADSTREAM, errbuf); + goto out; } chopprefix = drrb->drr_toname + strlen(drrb->drr_toname); } @@ -2784,39 +3088,41 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, chopprefix[0] == '\0'); /* - * Determine name of destination snapshot, store in zc_value. + * Determine name of destination snapshot. */ - (void) strcpy(zc.zc_value, tosnap); - (void) strlcat(zc.zc_value, chopprefix, sizeof (zc.zc_value)); + (void) strcpy(destsnap, tosnap); + (void) strlcat(destsnap, chopprefix, sizeof (destsnap)); free(cp); - if (!zfs_name_valid(zc.zc_value, ZFS_TYPE_SNAPSHOT)) { - zcmd_free_nvlists(&zc); - return (zfs_error(hdl, EZFS_INVALIDNAME, errbuf)); + if (!zfs_name_valid(destsnap, ZFS_TYPE_SNAPSHOT)) { + err = zfs_error(hdl, EZFS_INVALIDNAME, errbuf); + goto out; } /* - * Determine the name of the origin snapshot, store in zc_string. + * Determine the name of the origin snapshot. */ if (drrb->drr_flags & DRR_FLAG_CLONE) { - if (guid_to_name(hdl, zc.zc_value, - drrb->drr_fromguid, zc.zc_string) != 0) { - zcmd_free_nvlists(&zc); + if (guid_to_name(hdl, destsnap, + drrb->drr_fromguid, B_FALSE, origin) != 0) { zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "local origin for clone %s does not exist"), - zc.zc_value); - return (zfs_error(hdl, EZFS_NOENT, errbuf)); + destsnap); + err = zfs_error(hdl, EZFS_NOENT, errbuf); + goto out; } if (flags->verbose) - (void) printf("found clone origin %s\n", zc.zc_string); + (void) printf("found clone origin %s\n", origin); } else if (originsnap) { - (void) strncpy(zc.zc_string, originsnap, ZFS_MAXNAMELEN); + (void) strncpy(origin, originsnap, sizeof (origin)); if (flags->verbose) (void) printf("using provided clone origin %s\n", - zc.zc_string); + origin); } + boolean_t resuming = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & + DMU_BACKUP_FEATURE_RESUMING; stream_wantsnewfs = (drrb->drr_fromguid == 0 || - (drrb->drr_flags & DRR_FLAG_CLONE) || originsnap); + (drrb->drr_flags & DRR_FLAG_CLONE) || originsnap) && !resuming; if (stream_wantsnewfs) { /* @@ -2826,18 +3132,18 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, "cannot receive new filesystem stream")); - (void) strcpy(zc.zc_name, zc.zc_value); - cp = strrchr(zc.zc_name, '/'); + (void) strcpy(name, destsnap); + cp = strrchr(name, '/'); if (cp) *cp = '\0'; if (cp && - !zfs_dataset_exists(hdl, zc.zc_name, ZFS_TYPE_DATASET)) { - char suffix[ZFS_MAXNAMELEN]; - (void) strcpy(suffix, strrchr(zc.zc_value, '/')); - if (guid_to_name(hdl, zc.zc_name, parent_snapguid, - zc.zc_value) == 0) { - *strchr(zc.zc_value, '@') = '\0'; - (void) strcat(zc.zc_value, suffix); + !zfs_dataset_exists(hdl, name, ZFS_TYPE_DATASET)) { + char suffix[ZFS_MAX_DATASET_NAME_LEN]; + (void) strcpy(suffix, strrchr(destsnap, '/')); + if (guid_to_name(hdl, name, parent_snapguid, + B_FALSE, destsnap) == 0) { + *strchr(destsnap, '@') = '\0'; + (void) strcat(destsnap, suffix); } } } else { @@ -2848,8 +3154,8 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, (void) snprintf(errbuf, sizeof (errbuf), dgettext(TEXT_DOMAIN, "cannot receive incremental stream")); - (void) strcpy(zc.zc_name, zc.zc_value); - *strchr(zc.zc_name, '@') = '\0'; + (void) strcpy(name, destsnap); + *strchr(name, '@') = '\0'; /* * If the exact receive path was specified and this is the @@ -2858,65 +3164,67 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, */ if ((flags->isprefix || (*(chopprefix = drrb->drr_toname + strlen(sendfs)) != '\0' && *chopprefix != '@')) && - !zfs_dataset_exists(hdl, zc.zc_name, ZFS_TYPE_DATASET)) { - char snap[ZFS_MAXNAMELEN]; - (void) strcpy(snap, strchr(zc.zc_value, '@')); - if (guid_to_name(hdl, zc.zc_name, drrb->drr_fromguid, - zc.zc_value) == 0) { - *strchr(zc.zc_value, '@') = '\0'; - (void) strcat(zc.zc_value, snap); + !zfs_dataset_exists(hdl, name, ZFS_TYPE_DATASET)) { + char snap[ZFS_MAX_DATASET_NAME_LEN]; + (void) strcpy(snap, strchr(destsnap, '@')); + if (guid_to_name(hdl, name, drrb->drr_fromguid, + B_FALSE, destsnap) == 0) { + *strchr(destsnap, '@') = '\0'; + (void) strcat(destsnap, snap); } } } - (void) strcpy(zc.zc_name, zc.zc_value); - *strchr(zc.zc_name, '@') = '\0'; + (void) strcpy(name, destsnap); + *strchr(name, '@') = '\0'; - if (zfs_dataset_exists(hdl, zc.zc_name, ZFS_TYPE_DATASET)) { + if (zfs_dataset_exists(hdl, name, ZFS_TYPE_DATASET)) { + zfs_cmd_t zc = {"\0"}; zfs_handle_t *zhp; + (void) strcpy(zc.zc_name, name); + /* - * Destination fs exists. Therefore this should either - * be an incremental, or the stream specifies a new fs - * (full stream or clone) and they want us to blow it - * away (and have therefore specified -F and removed any - * snapshots). + * Destination fs exists. It must be one of these cases: + * - an incremental send stream + * - the stream specifies a new fs (full stream or clone) + * and they want us to blow away the existing fs (and + * have therefore specified -F and removed any snapshots) + * - we are resuming a failed receive. */ if (stream_wantsnewfs) { if (!flags->force) { - zcmd_free_nvlists(&zc); zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "destination '%s' exists\n" - "must specify -F to overwrite it"), - zc.zc_name); - return (zfs_error(hdl, EZFS_EXISTS, errbuf)); + "must specify -F to overwrite it"), name); + err = zfs_error(hdl, EZFS_EXISTS, errbuf); + goto out; } if (ioctl(hdl->libzfs_fd, ZFS_IOC_SNAPSHOT_LIST_NEXT, &zc) == 0) { - zcmd_free_nvlists(&zc); zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "destination has snapshots (eg. %s)\n" "must destroy them to overwrite it"), - zc.zc_name); - return (zfs_error(hdl, EZFS_EXISTS, errbuf)); + name); + err = zfs_error(hdl, EZFS_EXISTS, errbuf); + goto out; } } - if ((zhp = zfs_open(hdl, zc.zc_name, + if ((zhp = zfs_open(hdl, name, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME)) == NULL) { - zcmd_free_nvlists(&zc); - return (-1); + err = -1; + goto out; } if (stream_wantsnewfs && zhp->zfs_dmustats.dds_origin[0]) { - zcmd_free_nvlists(&zc); zfs_close(zhp); zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "destination '%s' is a clone\n" - "must destroy it to overwrite it"), - zc.zc_name); - return (zfs_error(hdl, EZFS_EXISTS, errbuf)); + "must destroy it to overwrite it"), name); + err = zfs_error(hdl, EZFS_EXISTS, errbuf); + goto out; } if (!flags->dryrun && zhp->zfs_type == ZFS_TYPE_FILESYSTEM && @@ -2925,16 +3233,28 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, clp = changelist_gather(zhp, ZFS_PROP_NAME, 0, 0); if (clp == NULL) { zfs_close(zhp); - zcmd_free_nvlists(&zc); - return (-1); + err = -1; + goto out; } if (changelist_prefix(clp) != 0) { changelist_free(clp); zfs_close(zhp); - zcmd_free_nvlists(&zc); - return (-1); + err = -1; + goto out; } } + + /* + * If we are resuming a newfs, set newfs here so that we will + * mount it if the recv succeeds this time. We can tell + * that it was a newfs on the first recv because the fs + * itself will be inconsistent (if the fs existed when we + * did the first recv, we would have received it into + * .../%recv). + */ + if (resuming && zfs_prop_get_int(zhp, ZFS_PROP_INCONSISTENT)) + newfs = B_TRUE; + zfs_close(zhp); } else { /* @@ -2944,12 +3264,13 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, * specified only the pool name (i.e. if the destination name * contained no slash character). */ - if (!stream_wantsnewfs || - (cp = strrchr(zc.zc_name, '/')) == NULL) { - zcmd_free_nvlists(&zc); + cp = strrchr(name, '/'); + + if (!stream_wantsnewfs || cp == NULL) { zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "destination '%s' does not exist"), zc.zc_name); - return (zfs_error(hdl, EZFS_NOENT, errbuf)); + "destination '%s' does not exist"), name); + err = zfs_error(hdl, EZFS_NOENT, errbuf); + goto out; } /* @@ -2959,44 +3280,34 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, *cp = '\0'; if (flags->isprefix && !flags->istail && !flags->dryrun && - create_parents(hdl, zc.zc_value, strlen(tosnap)) != 0) { - zcmd_free_nvlists(&zc); - return (zfs_error(hdl, EZFS_BADRESTORE, errbuf)); + create_parents(hdl, destsnap, strlen(tosnap)) != 0) { + err = zfs_error(hdl, EZFS_BADRESTORE, errbuf); + goto out; } newfs = B_TRUE; } - zc.zc_begin_record = drr_noswap->drr_u.drr_begin; - zc.zc_cookie = infd; - zc.zc_guid = flags->force; if (flags->verbose) { (void) printf("%s %s stream of %s into %s\n", flags->dryrun ? "would receive" : "receiving", drrb->drr_fromguid ? "incremental" : "full", - drrb->drr_toname, zc.zc_value); + drrb->drr_toname, destsnap); (void) fflush(stdout); } if (flags->dryrun) { - zcmd_free_nvlists(&zc); - return (recv_skip(hdl, infd, flags->byteswap)); + err = recv_skip(hdl, infd, flags->byteswap); + goto out; } - zc.zc_nvlist_dst = (uint64_t)(uintptr_t)prop_errbuf; - zc.zc_nvlist_dst_size = sizeof (prop_errbuf); - zc.zc_cleanup_fd = cleanup_fd; - zc.zc_action_handle = *action_handlep; - - err = ioctl_err = zfs_ioctl(hdl, ZFS_IOC_RECV, &zc); - ioctl_errno = errno; - prop_errflags = (zprop_errflags_t)zc.zc_obj; + err = ioctl_err = lzc_receive_one(destsnap, props, origin, + flags->force, flags->resumable, infd, drr_noswap, cleanup_fd, + &read_bytes, &errflags, action_handlep, &prop_errors); + ioctl_errno = ioctl_err; + prop_errflags = errflags; if (err == 0) { - nvlist_t *prop_errors; - VERIFY(0 == nvlist_unpack((void *)(uintptr_t)zc.zc_nvlist_dst, - zc.zc_nvlist_dst_size, &prop_errors, 0)); - nvpair_t *prop_err = NULL; while ((prop_err = nvlist_next_nvpair(prop_errors, @@ -3011,29 +3322,38 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, ZPROP_N_MORE_ERRORS) == 0) { trunc_prop_errs(intval); break; - } else { + } else if (snapname == NULL || finalsnap == NULL || + strcmp(finalsnap, snapname) == 0 || + strcmp(nvpair_name(prop_err), + zfs_prop_to_name(ZFS_PROP_REFQUOTA)) != 0) { + /* + * Skip the special case of, for example, + * "refquota", errors on intermediate + * snapshots leading up to a final one. + * That's why we have all of the checks above. + * + * See zfs_ioctl.c's extract_delay_props() for + * a list of props which can fail on + * intermediate snapshots, but shouldn't + * affect the overall receive. + */ (void) snprintf(tbuf, sizeof (tbuf), dgettext(TEXT_DOMAIN, "cannot receive %s property on %s"), - nvpair_name(prop_err), zc.zc_name); + nvpair_name(prop_err), name); zfs_setprop_error(hdl, prop, intval, tbuf); } } - nvlist_free(prop_errors); } - zc.zc_nvlist_dst = 0; - zc.zc_nvlist_dst_size = 0; - zcmd_free_nvlists(&zc); - if (err == 0 && snapprops_nvlist) { - zfs_cmd_t zc2 = {"\0"}; + zfs_cmd_t zc = {"\0"}; - (void) strcpy(zc2.zc_name, zc.zc_value); - zc2.zc_cookie = B_TRUE; /* received */ - if (zcmd_write_src_nvlist(hdl, &zc2, snapprops_nvlist) == 0) { - (void) zfs_ioctl(hdl, ZFS_IOC_SET_PROP, &zc2); - zcmd_free_nvlists(&zc2); + (void) strcpy(zc.zc_name, destsnap); + zc.zc_cookie = B_TRUE; /* received */ + if (zcmd_write_src_nvlist(hdl, &zc, snapprops_nvlist) == 0) { + (void) zfs_ioctl(hdl, ZFS_IOC_SET_PROP, &zc); + zcmd_free_nvlists(&zc); } } @@ -3045,7 +3365,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, */ avl_tree_t *local_avl; nvlist_t *local_nv, *fs; - cp = strchr(zc.zc_value, '@'); + cp = strchr(destsnap, '@'); /* * XXX Do this faster by just iterating over snaps in @@ -3053,7 +3373,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, * get a strange "does not exist" error message. */ *cp = '\0'; - if (gather_nvlist(hdl, zc.zc_value, NULL, NULL, B_FALSE, + if (gather_nvlist(hdl, destsnap, NULL, NULL, B_FALSE, &local_nv, &local_avl) == 0) { *cp = '@'; fs = fsavl_find(local_avl, drrb->drr_toguid, NULL); @@ -3063,7 +3383,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, if (fs != NULL) { if (flags->verbose) { (void) printf("snap %s already exists; " - "ignoring\n", zc.zc_value); + "ignoring\n", destsnap); } err = ioctl_err = recv_skip(hdl, infd, flags->byteswap); @@ -3075,22 +3395,22 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, if (ioctl_err != 0) { switch (ioctl_errno) { case ENODEV: - cp = strchr(zc.zc_value, '@'); + cp = strchr(destsnap, '@'); *cp = '\0'; zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "most recent snapshot of %s does not\n" - "match incremental source"), zc.zc_value); + "match incremental source"), destsnap); (void) zfs_error(hdl, EZFS_BADRESTORE, errbuf); *cp = '@'; break; case ETXTBSY: zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, "destination %s has been modified\n" - "since most recent snapshot"), zc.zc_name); + "since most recent snapshot"), name); (void) zfs_error(hdl, EZFS_BADRESTORE, errbuf); break; case EEXIST: - cp = strchr(zc.zc_value, '@'); + cp = strchr(destsnap, '@'); if (newfs) { /* it's the containing fs that exists */ *cp = '\0'; @@ -3099,15 +3419,18 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, "destination already exists")); (void) zfs_error_fmt(hdl, EZFS_EXISTS, dgettext(TEXT_DOMAIN, "cannot restore to %s"), - zc.zc_value); + destsnap); *cp = '@'; break; case EINVAL: + if (flags->resumable) + zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, + "kernel modules must be upgraded to " + "receive this stream.")); (void) zfs_error(hdl, EZFS_BADSTREAM, errbuf); break; case ECKSUM: - zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "invalid stream (checksum mismatch)")); + recv_ecksum_set_aux(hdl, destsnap, flags->resumable); (void) zfs_error(hdl, EZFS_BADSTREAM, errbuf); break; case ENOTSUP: @@ -3117,7 +3440,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, break; case EDQUOT: zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, - "destination %s space quota exceeded"), zc.zc_name); + "destination %s space quota exceeded"), name); (void) zfs_error(hdl, EZFS_NOSPC, errbuf); break; default: @@ -3130,12 +3453,12 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, * children of the target filesystem if we did a replication * receive (indicated by stream_avl being non-NULL). */ - cp = strchr(zc.zc_value, '@'); + cp = strchr(destsnap, '@'); if (cp && (ioctl_err == 0 || !newfs)) { zfs_handle_t *h; *cp = '\0'; - h = zfs_open(hdl, zc.zc_value, + h = zfs_open(hdl, destsnap, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME); if (h != NULL) { if (h->zfs_type == ZFS_TYPE_VOLUME) { @@ -3146,7 +3469,7 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, * for mounting and sharing later. */ if (top_zfs && *top_zfs == NULL) - *top_zfs = zfs_strdup(hdl, zc.zc_value); + *top_zfs = zfs_strdup(hdl, destsnap); } zfs_close(h); } @@ -3160,26 +3483,24 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, if (prop_errflags & ZPROP_ERR_NOCLEAR) { (void) fprintf(stderr, dgettext(TEXT_DOMAIN, "Warning: " - "failed to clear unreceived properties on %s"), - zc.zc_name); + "failed to clear unreceived properties on %s"), name); (void) fprintf(stderr, "\n"); } if (prop_errflags & ZPROP_ERR_NORESTORE) { (void) fprintf(stderr, dgettext(TEXT_DOMAIN, "Warning: " - "failed to restore original properties on %s"), - zc.zc_name); + "failed to restore original properties on %s"), name); (void) fprintf(stderr, "\n"); } - if (err || ioctl_err) - return (-1); - - *action_handlep = zc.zc_action_handle; + if (err || ioctl_err) { + err = -1; + goto out; + } if (flags->verbose) { char buf1[64]; char buf2[64]; - uint64_t bytes = zc.zc_cookie; + uint64_t bytes = read_bytes; time_t delta = time(NULL) - begin_time; if (delta == 0) delta = 1; @@ -3190,14 +3511,22 @@ zfs_receive_one(libzfs_handle_t *hdl, int infd, const char *tosnap, buf1, delta, buf2); } - return (0); + err = 0; +out: + if (prop_errors != NULL) + nvlist_free(prop_errors); + + if (newprops) + nvlist_free(props); + + return (err); } static int zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, const char *originsnap, recvflags_t *flags, int infd, const char *sendfs, nvlist_t *stream_nv, avl_tree_t *stream_avl, char **top_zfs, int cleanup_fd, - uint64_t *action_handlep) + uint64_t *action_handlep, const char *finalsnap) { int err; dmu_replay_record_t drr, drr_noswap; @@ -3281,7 +3610,7 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, } if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == DMU_SUBSTREAM) { - char nonpackage_sendfs[ZFS_MAXNAMELEN]; + char nonpackage_sendfs[ZFS_MAX_DATASET_NAME_LEN]; if (sendfs == NULL) { /* * We were not called from zfs_receive_package(). Get @@ -3289,14 +3618,16 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, */ char *cp; (void) strlcpy(nonpackage_sendfs, - drr.drr_u.drr_begin.drr_toname, ZFS_MAXNAMELEN); + drr.drr_u.drr_begin.drr_toname, + sizeof (nonpackage_sendfs)); if ((cp = strchr(nonpackage_sendfs, '@')) != NULL) *cp = '\0'; sendfs = nonpackage_sendfs; + VERIFY(finalsnap == NULL); } return (zfs_receive_one(hdl, infd, tosnap, originsnap, flags, &drr, &drr_noswap, sendfs, stream_nv, stream_avl, top_zfs, - cleanup_fd, action_handlep)); + cleanup_fd, action_handlep, finalsnap)); } else { assert(DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == DMU_COMPOUNDSTREAM); @@ -3309,7 +3640,8 @@ zfs_receive_impl(libzfs_handle_t *hdl, const char *tosnap, * Restores a backup of tosnap from the file descriptor specified by infd. * Return 0 on total success, -2 if some things couldn't be * destroyed/renamed/promoted, -1 if some things couldn't be received. - * (-1 will override -2). + * (-1 will override -2, if -1 and the resumable flag was specified the + * transfer can be resumed if the sending side supports it). */ int zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props, @@ -3373,7 +3705,7 @@ zfs_receive(libzfs_handle_t *hdl, const char *tosnap, nvlist_t *props, VERIFY(cleanup_fd >= 0); err = zfs_receive_impl(hdl, tosnap, originsnap, flags, infd, NULL, NULL, - stream_avl, &top_zfs, cleanup_fd, &action_handle); + stream_avl, &top_zfs, cleanup_fd, &action_handle, NULL); VERIFY(0 == close(cleanup_fd)); diff --git a/lib/libzfs/libzfs_util.c b/lib/libzfs/libzfs_util.c index 926ed4ed8aa9..8fe59a0c04ff 100644 --- a/lib/libzfs/libzfs_util.c +++ b/lib/libzfs/libzfs_util.c @@ -390,7 +390,7 @@ zfs_standard_error_fmt(libzfs_handle_t *hdl, int error, const char *fmt, ...) case ENOSPC: case EDQUOT: zfs_verror(hdl, EZFS_NOSPC, fmt, ap); - return (-1); + break; case EEXIST: zfs_error_aux(hdl, dgettext(TEXT_DOMAIN, diff --git a/lib/libzfs_core/libzfs_core.c b/lib/libzfs_core/libzfs_core.c index b706e6f6be88..0359e4284ac1 100644 --- a/lib/libzfs_core/libzfs_core.c +++ b/lib/libzfs_core/libzfs_core.c @@ -217,7 +217,7 @@ lzc_snapshot(nvlist_t *snaps, nvlist_t *props, nvlist_t **errlist) nvpair_t *elem; nvlist_t *args; int error; - char pool[MAXNAMELEN]; + char pool[ZFS_MAX_DATASET_NAME_LEN]; *errlist = NULL; @@ -269,7 +269,7 @@ lzc_destroy_snaps(nvlist_t *snaps, boolean_t defer, nvlist_t **errlist) nvpair_t *elem; nvlist_t *args; int error; - char pool[MAXNAMELEN]; + char pool[ZFS_MAX_DATASET_NAME_LEN]; /* determine the pool name */ elem = nvlist_next_nvpair(snaps, NULL); @@ -296,7 +296,7 @@ lzc_snaprange_space(const char *firstsnap, const char *lastsnap, nvlist_t *args; nvlist_t *result; int err; - char fs[MAXNAMELEN]; + char fs[ZFS_MAX_DATASET_NAME_LEN]; char *atp; /* determine the fs name */ @@ -361,7 +361,7 @@ lzc_exists(const char *dataset) int lzc_hold(nvlist_t *holds, int cleanup_fd, nvlist_t **errlist) { - char pool[MAXNAMELEN]; + char pool[ZFS_MAX_DATASET_NAME_LEN]; nvlist_t *args; nvpair_t *elem; int error; @@ -408,7 +408,7 @@ lzc_hold(nvlist_t *holds, int cleanup_fd, nvlist_t **errlist) int lzc_release(nvlist_t *holds, nvlist_t **errlist) { - char pool[MAXNAMELEN]; + char pool[ZFS_MAX_DATASET_NAME_LEN]; nvpair_t *elem; /* determine the pool name */ @@ -467,6 +467,13 @@ lzc_get_holds(const char *snapname, nvlist_t **holdsp) int lzc_send(const char *snapname, const char *from, int fd, enum lzc_send_flags flags) +{ + return (lzc_send_resume(snapname, from, fd, flags, 0, 0)); +} + +int +lzc_send_resume(const char *snapname, const char *from, int fd, + enum lzc_send_flags flags, uint64_t resumeobj, uint64_t resumeoff) { nvlist_t *args; int err; @@ -479,6 +486,10 @@ lzc_send(const char *snapname, const char *from, int fd, fnvlist_add_boolean(args, "largeblockok"); if (flags & LZC_SEND_FLAG_EMBED_DATA) fnvlist_add_boolean(args, "embedok"); + if (resumeobj != 0 || resumeoff != 0) { + fnvlist_add_uint64(args, "resume_object", resumeobj); + fnvlist_add_uint64(args, "resume_offset", resumeoff); + } err = lzc_ioctl(ZFS_IOC_SEND_NEW, snapname, args, NULL); nvlist_free(args); return (err); @@ -537,91 +548,258 @@ recv_read(int fd, void *buf, int ilen) } /* - * The simplest receive case: receive from the specified fd, creating the - * specified snapshot. Apply the specified properties a "received" properties - * (which can be overridden by locally-set properties). If the stream is a - * clone, its origin snapshot must be specified by 'origin'. The 'force' - * flag will cause the target filesystem to be rolled back or destroyed if - * necessary to receive. + * Linux adds ZFS_IOC_RECV_NEW for resumable streams and preserves the legacy + * ZFS_IOC_RECV user/kernel interface. The new interface supports all stream + * options but is currently only used for resumable streams. This way updated + * user space utilities will interoperate with older kernel modules. * - * Return 0 on success or an errno on failure. - * - * Note: this interface does not work on dedup'd streams - * (those with DMU_BACKUP_FEATURE_DEDUP). + * Non-Linux OpenZFS platforms have opted to modify the legacy interface. */ -int -lzc_receive(const char *snapname, nvlist_t *props, const char *origin, - boolean_t force, int fd) +static int +recv_impl(const char *snapname, nvlist_t *props, const char *origin, + boolean_t force, boolean_t resumable, int input_fd, + const dmu_replay_record_t *begin_record, int cleanup_fd, + uint64_t *read_bytes, uint64_t *errflags, uint64_t *action_handle, + nvlist_t **errors) { - /* - * The receive ioctl is still legacy, so we need to construct our own - * zfs_cmd_t rather than using zfsc_ioctl(). - */ - zfs_cmd_t zc = {"\0"}; - char *atp; - char *packed = NULL; - size_t size; dmu_replay_record_t drr; + char fsname[MAXPATHLEN]; + char *atp; int error; - ASSERT3S(g_refcount, >, 0); - - /* zc_name is name of containing filesystem */ - (void) strlcpy(zc.zc_name, snapname, sizeof (zc.zc_name)); - atp = strchr(zc.zc_name, '@'); + /* Set 'fsname' to the name of containing filesystem */ + (void) strlcpy(fsname, snapname, sizeof (fsname)); + atp = strchr(fsname, '@'); if (atp == NULL) return (EINVAL); *atp = '\0'; - /* if the fs does not exist, try its parent. */ - if (!lzc_exists(zc.zc_name)) { - char *slashp = strrchr(zc.zc_name, '/'); + /* If the fs does not exist, try its parent. */ + if (!lzc_exists(fsname)) { + char *slashp = strrchr(fsname, '/'); if (slashp == NULL) return (ENOENT); *slashp = '\0'; + } + /* + * The begin_record is normally a non-byteswapped BEGIN record. + * For resumable streams it may be set to any non-byteswapped + * dmu_replay_record_t. + */ + if (begin_record == NULL) { + error = recv_read(input_fd, &drr, sizeof (drr)); + if (error != 0) + return (error); + } else { + drr = *begin_record; } - /* zc_value is full name of the snapshot to create */ - (void) strlcpy(zc.zc_value, snapname, sizeof (zc.zc_value)); + if (resumable) { + nvlist_t *outnvl = NULL; + nvlist_t *innvl = fnvlist_alloc(); - if (props != NULL) { - /* zc_nvlist_src is props to set */ - packed = fnvlist_pack(props, &size); - zc.zc_nvlist_src = (uint64_t)(uintptr_t)packed; - zc.zc_nvlist_src_size = size; - } + fnvlist_add_string(innvl, "snapname", snapname); - /* zc_string is name of clone origin (if DRR_FLAG_CLONE) */ - if (origin != NULL) - (void) strlcpy(zc.zc_string, origin, sizeof (zc.zc_string)); + if (props != NULL) + fnvlist_add_nvlist(innvl, "props", props); - /* zc_begin_record is non-byteswapped BEGIN record */ - error = recv_read(fd, &drr, sizeof (drr)); - if (error != 0) - goto out; - zc.zc_begin_record = drr.drr_u.drr_begin; + if (origin != NULL && strlen(origin)) + fnvlist_add_string(innvl, "origin", origin); - /* zc_cookie is fd to read from */ - zc.zc_cookie = fd; + fnvlist_add_byte_array(innvl, "begin_record", + (uchar_t *) &drr, sizeof (drr)); - /* zc guid is force flag */ - zc.zc_guid = force; + fnvlist_add_int32(innvl, "input_fd", input_fd); - /* zc_cleanup_fd is unused */ - zc.zc_cleanup_fd = -1; + if (force) + fnvlist_add_boolean(innvl, "force"); - error = ioctl(g_fd, ZFS_IOC_RECV, &zc); - if (error != 0) - error = errno; + if (resumable) + fnvlist_add_boolean(innvl, "resumable"); + + if (cleanup_fd >= 0) + fnvlist_add_int32(innvl, "cleanup_fd", cleanup_fd); + + if (action_handle != NULL) + fnvlist_add_uint64(innvl, "action_handle", + *action_handle); + + error = lzc_ioctl(ZFS_IOC_RECV_NEW, fsname, innvl, &outnvl); + + if (error == 0 && read_bytes != NULL) + error = nvlist_lookup_uint64(outnvl, "read_bytes", + read_bytes); + + if (error == 0 && errflags != NULL) + error = nvlist_lookup_uint64(outnvl, "error_flags", + errflags); + + if (error == 0 && action_handle != NULL) + error = nvlist_lookup_uint64(outnvl, "action_handle", + action_handle); + + if (error == 0 && errors != NULL) { + nvlist_t *nvl; + error = nvlist_lookup_nvlist(outnvl, "errors", &nvl); + if (error == 0) + *errors = fnvlist_dup(nvl); + } + + fnvlist_free(innvl); + fnvlist_free(outnvl); + } else { + zfs_cmd_t zc = {"\0"}; + char *packed = NULL; + size_t size; + + ASSERT3S(g_refcount, >, 0); + + (void) strlcpy(zc.zc_name, fsname, sizeof (zc.zc_value)); + (void) strlcpy(zc.zc_value, snapname, sizeof (zc.zc_value)); + + if (props != NULL) { + packed = fnvlist_pack(props, &size); + zc.zc_nvlist_src = (uint64_t)(uintptr_t)packed; + zc.zc_nvlist_src_size = size; + } + + if (origin != NULL) + (void) strlcpy(zc.zc_string, origin, + sizeof (zc.zc_string)); + + ASSERT3S(drr.drr_type, ==, DRR_BEGIN); + zc.zc_begin_record = drr.drr_u.drr_begin; + zc.zc_guid = force; + zc.zc_cookie = input_fd; + zc.zc_cleanup_fd = -1; + zc.zc_action_handle = 0; + + if (cleanup_fd >= 0) + zc.zc_cleanup_fd = cleanup_fd; + + if (action_handle != NULL) + zc.zc_action_handle = *action_handle; + + zc.zc_nvlist_dst_size = 128 * 1024; + zc.zc_nvlist_dst = (uint64_t)(uintptr_t) + malloc(zc.zc_nvlist_dst_size); + + error = ioctl(g_fd, ZFS_IOC_RECV, &zc); + if (error != 0) { + error = errno; + } else { + if (read_bytes != NULL) + *read_bytes = zc.zc_cookie; + + if (errflags != NULL) + *errflags = zc.zc_obj; + + if (action_handle != NULL) + *action_handle = zc.zc_action_handle; + + if (errors != NULL) + VERIFY0(nvlist_unpack( + (void *)(uintptr_t)zc.zc_nvlist_dst, + zc.zc_nvlist_dst_size, errors, KM_SLEEP)); + } + + if (packed != NULL) + fnvlist_pack_free(packed, size); + free((void *)(uintptr_t)zc.zc_nvlist_dst); + } -out: - if (packed != NULL) - fnvlist_pack_free(packed, size); - free((void*)(uintptr_t)zc.zc_nvlist_dst); return (error); } +/* + * The simplest receive case: receive from the specified fd, creating the + * specified snapshot. Apply the specified properties as "received" properties + * (which can be overridden by locally-set properties). If the stream is a + * clone, its origin snapshot must be specified by 'origin'. The 'force' + * flag will cause the target filesystem to be rolled back or destroyed if + * necessary to receive. + * + * Return 0 on success or an errno on failure. + * + * Note: this interface does not work on dedup'd streams + * (those with DMU_BACKUP_FEATURE_DEDUP). + */ +int +lzc_receive(const char *snapname, nvlist_t *props, const char *origin, + boolean_t force, int fd) +{ + return (recv_impl(snapname, props, origin, force, B_FALSE, fd, + NULL, -1, NULL, NULL, NULL, NULL)); +} + +/* + * Like lzc_receive, but if the receive fails due to premature stream + * termination, the intermediate state will be preserved on disk. In this + * case, ECKSUM will be returned. The receive may subsequently be resumed + * with a resuming send stream generated by lzc_send_resume(). + */ +int +lzc_receive_resumable(const char *snapname, nvlist_t *props, const char *origin, + boolean_t force, int fd) +{ + return (recv_impl(snapname, props, origin, force, B_TRUE, fd, + NULL, -1, NULL, NULL, NULL, NULL)); +} + +/* + * Like lzc_receive, but allows the caller to read the begin record and then to + * pass it in. That could be useful if the caller wants to derive, for example, + * the snapname or the origin parameters based on the information contained in + * the begin record. + * The begin record must be in its original form as read from the stream, + * in other words, it should not be byteswapped. + * + * The 'resumable' parameter allows to obtain the same behavior as with + * lzc_receive_resumable. + */ +int +lzc_receive_with_header(const char *snapname, nvlist_t *props, + const char *origin, boolean_t force, boolean_t resumable, int fd, + const dmu_replay_record_t *begin_record) +{ + if (begin_record == NULL) + return (EINVAL); + return (recv_impl(snapname, props, origin, force, resumable, fd, + begin_record, -1, NULL, NULL, NULL, NULL)); +} + +/* + * Like lzc_receive, but allows the caller to pass all supported arguments + * and retrieve all values returned. The only additional input parameter + * is 'cleanup_fd' which is used to set a cleanup-on-exit file descriptor. + * + * The following parameters all provide return values. Several may be set + * in the failure case and will contain additional information. + * + * The 'read_bytes' value will be set to the total number of bytes read. + * + * The 'errflags' value will contain zprop_errflags_t flags which are + * used to describe any failures. + * + * The 'action_handle' is used to pass the handle for this guid/ds mapping. + * It should be set to zero on first call and will contain an updated handle + * on success, it should be passed in subsequent calls. + * + * The 'errors' nvlist contains an entry for each unapplied received + * property. Callers are responsible for freeing this nvlist. + */ +int lzc_receive_one(const char *snapname, nvlist_t *props, + const char *origin, boolean_t force, boolean_t resumable, int input_fd, + const dmu_replay_record_t *begin_record, int cleanup_fd, + uint64_t *read_bytes, uint64_t *errflags, uint64_t *action_handle, + nvlist_t **errors) +{ + return (recv_impl(snapname, props, origin, force, resumable, + input_fd, begin_record, cleanup_fd, read_bytes, errflags, + action_handle, errors)); +} + /* * Roll back this filesystem or volume to its most recent snapshot. * If snapnamebuf is not NULL, it will be filled in with the name @@ -664,7 +842,7 @@ lzc_bookmark(nvlist_t *bookmarks, nvlist_t **errlist) { nvpair_t *elem; int error; - char pool[MAXNAMELEN]; + char pool[ZFS_MAX_DATASET_NAME_LEN]; /* determine the pool name */ elem = nvlist_next_nvpair(bookmarks, NULL); @@ -726,7 +904,7 @@ lzc_destroy_bookmarks(nvlist_t *bmarks, nvlist_t **errlist) { nvpair_t *elem; int error; - char pool[MAXNAMELEN]; + char pool[ZFS_MAX_DATASET_NAME_LEN]; /* determine the pool name */ elem = nvlist_next_nvpair(bmarks, NULL); diff --git a/lib/libzpool/Makefile.am b/lib/libzpool/Makefile.am index 9bb1a8eea3fa..e73268b737b0 100644 --- a/lib/libzpool/Makefile.am +++ b/lib/libzpool/Makefile.am @@ -22,6 +22,8 @@ KERNEL_C = \ zfs_comutil.c \ zfs_deleg.c \ zfs_fletcher.c \ + zfs_fletcher_intel.c \ + zfs_fletcher_sse.c \ zfs_namecheck.c \ zfs_prop.c \ zfs_uio.c \ @@ -91,6 +93,11 @@ KERNEL_C = \ vdev_missing.c \ vdev_queue.c \ vdev_raidz.c \ + vdev_raidz_math.c \ + vdev_raidz_math_scalar.c \ + vdev_raidz_math_sse2.c \ + vdev_raidz_math_ssse3.c \ + vdev_raidz_math_avx2.c \ vdev_root.c \ zap.c \ zap_leaf.c \ @@ -119,7 +126,8 @@ nodist_libzpool_la_SOURCES = \ libzpool_la_LIBADD = \ $(top_builddir)/lib/libunicode/libunicode.la \ $(top_builddir)/lib/libuutil/libuutil.la \ - $(top_builddir)/lib/libnvpair/libnvpair.la + $(top_builddir)/lib/libnvpair/libnvpair.la \ + $(top_builddir)/lib/libicp/libicp.la libzpool_la_LIBADD += $(ZLIB) libzpool_la_LDFLAGS = -version-info 2:0:0 diff --git a/lib/libzpool/kernel.c b/lib/libzpool/kernel.c index 89e474c65d44..a68911451886 100644 --- a/lib/libzpool/kernel.c +++ b/lib/libzpool/kernel.c @@ -40,6 +40,8 @@ #include #include #include +#include +#include /* * Emulation of kernel services in userland. @@ -1112,9 +1114,96 @@ lowbit64(uint64_t i) return (h); } +/* + * Find highest one bit set. + * Returns bit number + 1 of highest bit that is set, otherwise returns 0. + * High order bit is 31 (or 63 in _LP64 kernel). + */ +int +highbit(ulong_t i) +{ +register int h = 1; + + if (i == 0) + return (0); +#ifdef _LP64 + if (i & 0xffffffff00000000ul) { + h += 32; i >>= 32; + } +#endif + if (i & 0xffff0000) { + h += 16; i >>= 16; + } + if (i & 0xff00) { + h += 8; i >>= 8; + } + if (i & 0xf0) { + h += 4; i >>= 4; + } + if (i & 0xc) { + h += 2; i >>= 2; + } + if (i & 0x2) { + h += 1; + } + return (h); +} + +/* + * Find lowest one bit set. + * Returns bit number + 1 of lowest bit that is set, otherwise returns 0. + * Low order bit is 0. + */ +int +lowbit(ulong_t i) +{ + register int h = 1; + + if (i == 0) + return (0); + +#ifdef _LP64 + if (!(i & 0xffffffff)) { + h += 32; i >>= 32; + } +#endif + if (!(i & 0xffff)) { + h += 16; i >>= 16; + } + if (!(i & 0xff)) { + h += 8; i >>= 8; + } + if (!(i & 0xf)) { + h += 4; i >>= 4; + } + if (!(i & 0x3)) { + h += 2; i >>= 2; + } + if (!(i & 0x1)) { + h += 1; + } + return (h); +} static int random_fd = -1, urandom_fd = -1; +void +random_init(void) +{ + VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1); + VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1); +} + +void +random_fini(void) +{ + close(random_fd); + close(urandom_fd); + + random_fd = -1; + urandom_fd = -1; +} + static int random_get_bytes_common(uint8_t *ptr, size_t len, int fd) { @@ -1227,31 +1316,32 @@ kernel_init(int mode) (void) snprintf(hw_serial, sizeof (hw_serial), "%ld", (mode & FWRITE) ? get_system_hostid() : 0); - VERIFY((random_fd = open("/dev/random", O_RDONLY)) != -1); - VERIFY((urandom_fd = open("/dev/urandom", O_RDONLY)) != -1); + random_init(); + VERIFY0(uname(&hw_utsname)); thread_init(); system_taskq_init(); + icp_init(); spa_init(mode); + fletcher_4_init(); + tsd_create(&rrw_tsd_key, rrw_tsd_destroy); } void kernel_fini(void) { + fletcher_4_fini(); spa_fini(); + icp_fini(); system_taskq_fini(); thread_fini(); - close(random_fd); - close(urandom_fd); - - random_fd = -1; - urandom_fd = -1; + random_fini(); } uid_t @@ -1302,6 +1392,12 @@ zfs_secpolicy_destroy_perms(const char *name, cred_t *cr) return (0); } +int +secpolicy_zfs(const cred_t *cr) +{ + return (0); +} + ksiddomain_t * ksid_lookupdomain(const char *dom) { @@ -1398,6 +1494,8 @@ spl_fstrans_check(void) return (0); } +void *zvol_tag = "zvol_tag"; + void zvol_create_minors(spa_t *spa, const char *name, boolean_t async) { diff --git a/lib/libzpool/taskq.c b/lib/libzpool/taskq.c index 1d50d0edc242..e42e4261f8d7 100644 --- a/lib/libzpool/taskq.c +++ b/lib/libzpool/taskq.c @@ -34,26 +34,6 @@ int taskq_now; taskq_t *system_taskq; #define TASKQ_ACTIVE 0x00010000 -#define TASKQ_NAMELEN 31 - -struct taskq { - char tq_name[TASKQ_NAMELEN + 1]; - kmutex_t tq_lock; - krwlock_t tq_threadlock; - kcondvar_t tq_dispatch_cv; - kcondvar_t tq_wait_cv; - kthread_t **tq_threadlist; - int tq_flags; - int tq_active; - int tq_nthreads; - int tq_nalloc; - int tq_minalloc; - int tq_maxalloc; - kcondvar_t tq_maxalloc_cv; - int tq_maxalloc_wait; - taskq_ent_t *tq_freelist; - taskq_ent_t tq_task; -}; static taskq_ent_t * task_alloc(taskq_t *tq, int tqflags) diff --git a/man/man1/Makefile.am b/man/man1/Makefile.am index 113cd0d80e44..5f18d1ed6388 100644 --- a/man/man1/Makefile.am +++ b/man/man1/Makefile.am @@ -1,4 +1,4 @@ -dist_man_MANS = zhack.1 zpios.1 ztest.1 +dist_man_MANS = zhack.1 zpios.1 ztest.1 raidz_test.1 EXTRA_DIST = cstyle.1 install-data-local: diff --git a/man/man1/raidz_test.1 b/man/man1/raidz_test.1 new file mode 100644 index 000000000000..90d858d5bb40 --- /dev/null +++ b/man/man1/raidz_test.1 @@ -0,0 +1,97 @@ +'\" t +.\" +.\" CDDL HEADER START +.\" +.\" The contents of this file are subject to the terms of the +.\" Common Development and Distribution License (the "License"). +.\" You may not use this file except in compliance with the License. +.\" +.\" You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE +.\" or http://www.opensolaris.org/os/licensing. +.\" See the License for the specific language governing permissions +.\" and limitations under the License. +.\" +.\" When distributing Covered Code, include this CDDL HEADER in each +.\" file and include the License file at usr/src/OPENSOLARIS.LICENSE. +.\" If applicable, add the following below this CDDL HEADER, with the +.\" fields enclosed by brackets "[]" replaced with your own identifying +.\" information: Portions Copyright [yyyy] [name of copyright owner] +.\" +.\" CDDL HEADER END +.\" +.\" +.\" Copyright (c) 2016 Gvozden Nešković. All rights reserved. +.\" +.TH raidz_test 1 "2016" "ZFS on Linux" "User Commands" + +.SH NAME +\fBraidz_test\fR \- raidz implementation verification and bencmarking tool +.SH SYNOPSIS +.LP +.BI "raidz_test " +.SH DESCRIPTION +.LP +This manual page documents briefly the \fBraidz_test\fR command. +.LP +Purpose of this tool is to run all supported raidz implementation and verify +results of all methods. Tool also contains a parameter sweep option where all +parameters affecting RAIDZ block are verified (like ashift size, data offset, +data size, etc...). +The tool also supports a benchmarking mode using -B option. +.SH OPTION +.HP +.BI "\-h" "" +.IP +Print a help summary. +.HP +.BI "\-a" " ashift (default: 9)" +.IP +Ashift value. +.HP +.BI "\-o" " zio_off_shift" " (default: 0)" +.IP +Zio offset for raidz block. Offset value is 1 << (zio_off_shift) +.HP +.BI "\-d" " raidz_data_disks" " (default: 8)" +.IP +Number of raidz data disks to use. Additional disks for parity will be used +during testing. +.HP +.BI "\-s" " zio_size_shift" " (default: 19)" +.IP +Size of data for raidz block. Size is 1 << (zio_size_shift). +.HP +.BI "\-S(weep)" +.IP +Sweep parameter space while verifying the raidz implementations. This option +will exhaust all most of valid values for -a -o -d -s options. Runtime using +this option will be long. +.HP +.BI "\-t(imeout)" +.IP +Wall time for sweep test in seconds. The actual runtime could be longer. +.HP +.BI "\-B(enchmark)" +.IP +This options starts the benchmark mode. All implementations are benchmarked +using increasing per disk data size. Results are given as throughput per disk, +measured in MiB/s. +.HP +.BI "\-v(erbose)" +.IP +Increase verbosity. +.HP +.BI "\-T(est the test)" +.IP +Debugging option. When this option is specified tool is supposed to fail +all tests. This is to check if tests would properly verify bit-exactness. +.HP +.BI "\-D(ebug)" +.IP +Debugging option. Specify to attach gdb when SIGSEGV or SIGABRT are received. +.HP + +.SH "SEE ALSO" +.BR "ztest (1)" +.SH "AUTHORS" +vdev_raidz, created for ZFS on Linux by Gvozden Nešković diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5 index 20022461ca33..41fc20debc87 100644 --- a/man/man5/zfs-module-parameters.5 +++ b/man/man5/zfs-module-parameters.5 @@ -30,7 +30,8 @@ Description of the different parameters to the ZFS module. \fBl2arc_feed_again\fR (int) .ad .RS 12n -Turbo L2ARC warmup +Turbo L2ARC warm-up. When the L2ARC is cold the fill interval will be set as +fast as possible. .sp Use \fB1\fR for yes (default) and \fB0\fR to disable. .RE @@ -41,7 +42,8 @@ Use \fB1\fR for yes (default) and \fB0\fR to disable. \fBl2arc_feed_min_ms\fR (ulong) .ad .RS 12n -Min feed interval in milliseconds +Min feed interval in milliseconds. Requires \fBl2arc_feed_again=1\fR and only +applicable in related situations. .sp Default value: \fB200\fR. .RE @@ -63,7 +65,8 @@ Default value: \fB1\fR. \fBl2arc_headroom\fR (ulong) .ad .RS 12n -Number of max device writes to precache +How far through the ARC lists to search for L2ARC cacheable content, expressed +as a multiplier of \fBl2arc_write_max\fR .sp Default value: \fB2\fR. .RE @@ -74,7 +77,8 @@ Default value: \fB2\fR. \fBl2arc_headroom_boost\fR (ulong) .ad .RS 12n -Compressed l2arc_headroom multiplier +Scales \fBl2arc_headroom\fR by this percentage when L2ARC contents are being +successfully compressed before writing. A value of 100 disables this feature. .sp Default value: \fB200\fR. .RE @@ -110,7 +114,8 @@ Use \fB1\fR for yes and \fB0\fR for no (default). \fBl2arc_noprefetch\fR (int) .ad .RS 12n -Skip caching prefetched buffers +Do not write buffers to L2ARC if they were prefetched but not used by +applications .sp Use \fB1\fR for yes (default) and \fB0\fR to disable. .RE @@ -132,7 +137,8 @@ Use \fB1\fR for yes and \fB0\fR for no (default). \fBl2arc_write_boost\fR (ulong) .ad .RS 12n -Extra write bytes during device warmup +Cold L2ARC devices will have \fBl2arc_write_nax\fR increased by this amount +while they remain cold. .sp Default value: \fB8,388,608\fR. .RE @@ -266,7 +272,7 @@ configuration. Pool administrators who understand the factors involved may wish to specify a more realistic inflation factor, particularly if they operate close to quota or capacity limits. .sp -Default value: 24 +Default value: \fB24\fR. .RE .sp @@ -283,7 +289,7 @@ blocks in the pool for verification. If this parameter is set to 0, the traversal skips non-metadata blocks. It can be toggled once the import has started to stop or start the traversal of non-metadata blocks. .sp -Default value: 1 +Default value: \fB1\fR. .RE .sp @@ -300,7 +306,7 @@ blocks in the pool for verification. If this parameter is set to 0, the traversal is not performed. It can be toggled once the import has started to stop or start the traversal. .sp -Default value: 1 +Default value: \fB1\fR. .RE .sp @@ -312,7 +318,7 @@ Default value: 1 Maximum concurrent I/Os during the traversal performed during an "extreme rewind" (\fB-X\fR) pool import. .sp -Default value: 10000 +Default value: \fB10000\fR. .RE .sp @@ -328,7 +334,7 @@ It also limits the worst-case time to allocate space. If we have less than this amount of free space, most ZPL operations (e.g. write, create) will return ENOSPC. .sp -Default value: 5 +Default value: \fB5\fR. .RE .sp @@ -375,6 +381,37 @@ Min time before an active prefetch stream can be reclaimed Default value: \fB2\fR. .RE +.sp +.ne 2 +.na +\fBzfs_arc_dnode_limit\fR (ulong) +.ad +.RS 12n +When the number of bytes consumed by dnodes in the ARC exceeds this number of +bytes, try to unpin some of it in response to demand for non-metadata. This +value acts as a floor to the amount of dnode metadata. + +See also \fBzfs_arc_meta_prune\fR which serves a similar purpose but is used +when the amount of metadata in the ARC exceeds \fBzfs_arc_meta_limit\fR rather +than in response to overall demand for non-metadata. + +.sp +Default value: \fB10% of zfs_arc_meta_limit\fR. +.RE + +.sp +.ne 2 +.na +\fBzfs_arc_dnode_reduce_percent\fR (ulong) +.ad +.RS 12n +Percentage of ARC dnodes to try to scan in response to demand for non-metadata +when the number of bytes consumed by dnodes exceeds \fBzfs_arc_dnode_limit\fB. + +.sp +Default value: \fB10% of the number of dnodes in the ARC\fR. +.RE + .sp .ne 2 .na @@ -410,7 +447,8 @@ Default value: \fB10\fR. \fBzfs_arc_grow_retry\fR (int) .ad .RS 12n -Seconds before growing arc size +After a memory pressure event the ARC will wait this many seconds before trying +to resume growth .sp Default value: \fB5\fR. .RE @@ -433,7 +471,12 @@ Default value: \fB10\fR. \fBzfs_arc_max\fR (ulong) .ad .RS 12n -Max arc size +Max arc size of ARC in bytes. If set to 0 then it will consume 1/2 of system +RAM. This value must be at least 67108864 (64 megabytes). +.sp +This value can be changed dynamically with some caveats. It cannot be set back +to 0 while running and reducing it below the current ARC size will not cause +the ARC to shrink without memory pressure to induce shrinking. .sp Default value: \fB0\fR. .RE @@ -450,6 +493,9 @@ be reclaimed even if the overall arc_c_max has not been reached. This value defaults to 0 which indicates that 3/4 of the ARC may be used for meta data. .sp +This value my be changed dynamically except that it cannot be set back to 0 +for 3/4 of the ARC; it must be set to an explicit value. +.sp Default value: \fB0\fR. .RE @@ -513,9 +559,10 @@ Default value: \fB100\fR. \fBzfs_arc_min_prefetch_lifespan\fR (int) .ad .RS 12n -Min life of prefetch block +Minimum time prefetched blocks are locked in the ARC, specified in jiffies. +A value of 0 will default to 1 second. .sp -Default value: \fB100\fR. +Default value: \fB0\fR. .RE .sp @@ -529,7 +576,7 @@ of lists for both data and meta data objects. Locking is performed at the level of these "sub-lists". This parameters controls the number of sub-lists per ARC state. .sp -Default value: 1 or the number of on-online CPUs, whichever is greater +Default value: \fR1\fB or the number of online CPUs, whichever is greater .RE .sp @@ -652,7 +699,8 @@ Default value: \fB4M\fR. \fBzfs_dbuf_state_index\fR (int) .ad .RS 12n -Calculate arc header index +This feature is currently unused. It is normally used for controlling what +reporting is available under /proc/spl/kstat/zfs. .sp Default value: \fB0\fR. .RE @@ -663,7 +711,7 @@ Default value: \fB0\fR. \fBzfs_deadman_enabled\fR (int) .ad .RS 12n -Enable deadman timer +Enable deadman timer. See description below. .sp Use \fB1\fR for yes (default) and \fB0\fR to disable. .RE @@ -785,7 +833,7 @@ time, and will be ignored if \fBzfs_dirty_data_max\fR is later changed. The parameter \fBzfs_dirty_data_max_max\fR takes precedence over this one. See the section "ZFS TRANSACTION DELAY". .sp -Default value: 25 +Default value: \fN25\fR. .RE .sp @@ -813,6 +861,27 @@ Start syncing out a transaction group if there is at least this much dirty data. Default value: \fB67,108,864\fR. .RE +.sp +.ne 2 +.na +\fBzfs_fletcher_4_impl\fR (string) +.ad +.RS 12n +Select a fletcher 4 implementation. +.sp +Supported selectors are: \fBfastest\fR, \fBscalar\fR, \fBsse2\fR, \fBssse3\fR, +and \fBavx2\fR. All of the selectors except \fBfastest\fR and \fBscalar\fR +require instruction set extensions to be available and will only appear if ZFS +detects that they are present at runtime. If multiple implementations of +fletcher 4 are available, the \fBfastest\fR will be chosen using a micro +benchmark. Selecting \fBscalar\fR results in the original CPU based calculation +being used. Selecting any option other than \fBfastest\fR and \fBscalar\fR +results in vector instructions from the respective CPU instruction set being +used. +.sp +Default value: \fBfastest\fR. +.RE + .sp .ne 2 .na @@ -841,7 +910,7 @@ Default value: \fB100,000\fR. \fBzfs_vdev_async_read_max_active\fR (int) .ad .RS 12n -Maxium asynchronous read I/Os active to each device. +Maximum asynchronous read I/Os active to each device. See the section "ZFS I/O SCHEDULER". .sp Default value: \fB3\fR. @@ -895,7 +964,7 @@ Default value: \fB30\fR. \fBzfs_vdev_async_write_max_active\fR (int) .ad .RS 12n -Maxium asynchronous write I/Os active to each device. +Maximum asynchronous write I/Os active to each device. See the section "ZFS I/O SCHEDULER". .sp Default value: \fB10\fR. @@ -932,7 +1001,7 @@ Default value: \fB1,000\fR. \fBzfs_vdev_scrub_max_active\fR (int) .ad .RS 12n -Maxium scrub I/Os active to each device. +Maximum scrub I/Os active to each device. See the section "ZFS I/O SCHEDULER". .sp Default value: \fB2\fR. @@ -956,7 +1025,7 @@ Default value: \fB1\fR. \fBzfs_vdev_sync_read_max_active\fR (int) .ad .RS 12n -Maxium synchronous read I/Os active to each device. +Maximum synchronous read I/Os active to each device. See the section "ZFS I/O SCHEDULER". .sp Default value: \fB10\fR. @@ -980,7 +1049,7 @@ Default value: \fB10\fR. \fBzfs_vdev_sync_write_max_active\fR (int) .ad .RS 12n -Maxium synchronous write I/Os active to each device. +Maximum synchronous write I/Os active to each device. See the section "ZFS I/O SCHEDULER". .sp Default value: \fB10\fR. @@ -1125,7 +1194,8 @@ Default value: \fB0\fR. \fBzfs_free_min_time_ms\fR (int) .ad .RS 12n -Min millisecs to free per txg +During a \fRzfs destroy\fB operation using \fRfeature@async_destroy\fB a minimum +of this much time will be spent working on freeing blocks per txg. .sp Default value: \fB1,000\fR. .RE @@ -1136,7 +1206,8 @@ Default value: \fB1,000\fR. \fBzfs_immediate_write_sz\fR (long) .ad .RS 12n -Largest data block to write to zil +Largest data block to write to zil. Larger blocks will be treated as if the +dataset being written to had the property setting \fRlogbias=throughput\fB. .sp Default value: \fB32,768\fR. .RE @@ -1191,7 +1262,7 @@ Default value: \fB70\fR. .ad .RS 12n Metaslab groups are considered eligible for allocations if their -fragmenation metric (measured as a percentage) is less than or equal to +fragmentation metric (measured as a percentage) is less than or equal to this value. If a metaslab group exceeds this threshold then it will be skipped unless all metaslab groups within the metaslab class have also crossed this threshold. @@ -1231,7 +1302,8 @@ Default value: \fB0\fR. \fBzfs_no_scrub_io\fR (int) .ad .RS 12n -Set for no scrub I/O +Set for no scrub I/O. This results in scrubs not actually scrubbing data and +simply doing a metadata crawl of the pool instead. .sp Use \fB1\fR for yes and \fB0\fR for no (default). .RE @@ -1242,7 +1314,7 @@ Use \fB1\fR for yes and \fB0\fR for no (default). \fBzfs_no_scrub_prefetch\fR (int) .ad .RS 12n -Set for no scrub prefetching +Set to disable block prefetching for scrubs. .sp Use \fB1\fR for yes and \fB0\fR for no (default). .RE @@ -1253,7 +1325,8 @@ Use \fB1\fR for yes and \fB0\fR for no (default). \fBzfs_nocacheflush\fR (int) .ad .RS 12n -Disable cache flushes +Disable cache flush operations on disks when writing. Beware, this may cause +corruption if disks re-order writes. .sp Use \fB1\fR for yes and \fB0\fR for no (default). .RE @@ -1275,7 +1348,8 @@ Use \fB1\fR for yes (default) and \fB0\fR to disable. \fBzfs_pd_bytes_max\fR (int) .ad .RS 12n -The number of bytes which should be prefetched. +The number of bytes which should be prefetched during a pool traversal +(eg: \fRzfs send\fB or other data crawling operations) .sp Default value: \fB52,428,800\fR. .RE @@ -1311,9 +1385,10 @@ Default value: \fB1,048,576\fR. \fBzfs_read_history\fR (int) .ad .RS 12n -Historic statistics for the last N reads +Historic statistics for the last N reads will be available in +\fR/proc/spl/kstat/zfs/POOLNAME/reads\fB .sp -Default value: \fB0\fR. +Default value: \fB0\fR (no data is kept). .RE .sp @@ -1358,7 +1433,8 @@ Default value: \fB2\fR. \fBzfs_resilver_min_time_ms\fR (int) .ad .RS 12n -Min millisecs to resilver per txg +Resilvers are processed by the sync thread. While resilvering it will spend +at least this much time working on a resilver between txg flushes. .sp Default value: \fB3,000\fR. .RE @@ -1383,7 +1459,8 @@ Default value: \fB50\fR. \fBzfs_scan_min_time_ms\fR (int) .ad .RS 12n -Min millisecs to scrub per txg +Scrubs are processed by the sync thread. While scrubbing it will spend +at least this much time working on a scrub between txg flushes. .sp Default value: \fB1,000\fR. .RE @@ -1407,7 +1484,7 @@ Default value: \fB4\fR. \fBzfs_send_corrupt_data\fR (int) .ad .RS 12n -Allow to send corrupt data (ignore read/checksum errors when sending data) +Allow sending of corrupt data (ignore read/checksum errors when sending data) .sp Use \fB1\fR for yes and \fB0\fR for no (default). .RE @@ -1418,7 +1495,7 @@ Use \fB1\fR for yes and \fB0\fR for no (default). \fBzfs_sync_pass_deferred_free\fR (int) .ad .RS 12n -Defer frees starting in this pass +Flushing of data to disk is done in passes. Defer frees starting in this pass .sp Default value: \fB2\fR. .RE @@ -1440,7 +1517,7 @@ Default value: \fB5\fR. \fBzfs_sync_pass_rewrite\fR (int) .ad .RS 12n -Rewrite new bps starting in this pass +Rewrite new block pointers starting in this pass .sp Default value: \fB2\fR. .RE @@ -1451,7 +1528,8 @@ Default value: \fB2\fR. \fBzfs_top_maxinflight\fR (int) .ad .RS 12n -Max I/Os per top-level vdev during scrub or resilver operations. +Max concurrent I/Os per top-level vdev (mirrors or raidz arrays) allowed during +scrub or resilver operations. .sp Default value: \fB32\fR. .RE @@ -1462,7 +1540,8 @@ Default value: \fB32\fR. \fBzfs_txg_history\fR (int) .ad .RS 12n -Historic statistics for the last N txgs +Historic statistics for the last N txgs will be available in +\fR/proc/spl/kstat/zfs/POOLNAME/txgs\fB .sp Default value: \fB0\fR. .RE @@ -1473,7 +1552,7 @@ Default value: \fB0\fR. \fBzfs_txg_timeout\fR (int) .ad .RS 12n -Max seconds worth of delta per txg +Flush dirty data to disk at least every N seconds (maximum txg duration) .sp Default value: \fB5\fR. .RE @@ -1497,7 +1576,7 @@ Default value: \fB131,072\fR. .RS 12n Shift size to inflate reads too .sp -Default value: \fB16\fR. +Default value: \fB16\fR (effectively 65536). .RE .sp @@ -1506,7 +1585,10 @@ Default value: \fB16\fR. \fBzfs_vdev_cache_max\fR (int) .ad .RS 12n -Inflate reads small than max +Inflate reads small than this value to meet the \fBzfs_vdev_cache_bshift\fR +size. +.sp +Default value: \fB16384\fR. .RE .sp @@ -1515,7 +1597,10 @@ Inflate reads small than max \fBzfs_vdev_cache_size\fR (int) .ad .RS 12n -Total size of the per-disk cache +Total size of the per-disk cache in bytes. +.sp +Currently this feature is disabled as it has been found to not be helpful +for performance and in some cases harmful. .sp Default value: \fB0\fR. .RE @@ -1596,7 +1681,8 @@ Default value: \fB1\fR. \fBzfs_vdev_read_gap_limit\fR (int) .ad .RS 12n -Aggregate read I/O over gap +Aggregate read I/O operations if the gap on-disk between them is within this +threshold. .sp Default value: \fB32,768\fR. .RE @@ -1607,7 +1693,7 @@ Default value: \fB32,768\fR. \fBzfs_vdev_scheduler\fR (charp) .ad .RS 12n -I/O scheduler +Set the Linux I/O scheduler on whole disk vdevs to this scheduler .sp Default value: \fBnoop\fR. .RE @@ -1623,13 +1709,41 @@ Aggregate write I/O over gap Default value: \fB4,096\fR. .RE +.sp +.ne 2 +.na +\fBzfs_vdev_raidz_impl\fR (string) +.ad +.RS 12n +Parameter for selecting raidz parity implementation to use. + +Options marked (always) below may be selected on module load as they are +supported on all systems. +The remaining options may only be set after the module is loaded, as they +are available only if the implementations are compiled in and supported +on the running system. + +Once the module is loaded, the content of +/sys/module/zfs/parameters/zfs_vdev_raidz_impl will show available options +with the currently selected one enclosed in []. +Possible options are: + fastest - (always) implementation selected using built-in benchmark + original - (always) original raidz implementation + scalar - (always) scalar raidz implementation + sse2 - implementation using SSE2 instruction set (64bit x86 only) + ssse3 - implementation using SSSE3 instruction set (64bit x86 only) + avx2 - implementation using AVX2 instruction set (64bit x86 only) +.sp +Default value: \fBfastest\fR. +.RE + .sp .ne 2 .na \fBzfs_zevent_cols\fR (int) .ad .RS 12n -Max event column width +When zevents are logged to the console use this as the word wrap width. .sp Default value: \fB80\fR. .RE @@ -1651,7 +1765,9 @@ Use \fB1\fR for yes and \fB0\fR for no (default). \fBzfs_zevent_len_max\fR (int) .ad .RS 12n -Max event queue length +Max event queue length. A value of 0 will result in a calculated value which +increases with the number of CPUs in the system (minimum 64 events). Events +in the queue can be viewed with the \fBzpool events\fR command. .sp Default value: \fB0\fR. .RE @@ -1662,7 +1778,8 @@ Default value: \fB0\fR. \fBzil_replay_disable\fR (int) .ad .RS 12n -Disable intent logging replay +Disable intent logging replay. Can be disabled for recovery from corrupted +ZIL .sp Use \fB1\fR for yes and \fB0\fR for no (default). .RE @@ -1684,7 +1801,9 @@ Default value: \fB1,048,576\fR. \fBzio_delay_max\fR (int) .ad .RS 12n -Max zio millisecond delay before posting event +A zevent will be logged if a ZIO operation takes more than N milliseconds to +complete. Note that this is only a logging facility, not a timeout on +operations. .sp Default value: \fB30,000\fR. .RE @@ -1723,7 +1842,8 @@ Default value: \fB75\fR. \fBzvol_inhibit_dev\fR (uint) .ad .RS 12n -Do not create zvol device nodes +Do not create zvol device nodes. This may slightly improve startup time on +systems with a very large number of zvols. .sp Use \fB1\fR for yes and \fB0\fR for no (default). .RE @@ -1734,7 +1854,7 @@ Use \fB1\fR for yes and \fB0\fR for no (default). \fBzvol_major\fR (uint) .ad .RS 12n -Major number for zvol device +Major number for zvol block devices .sp Default value: \fB230\fR. .RE @@ -1745,7 +1865,9 @@ Default value: \fB230\fR. \fBzvol_max_discard_blocks\fR (ulong) .ad .RS 12n -Max number of blocks to discard at once +Discard (aka TRIM) operations done on zvols will be done in batches of this +many blocks, where block size is determined by the \fBvolblocksize\fR property +of a zvol. .sp Default value: \fB16,384\fR. .RE diff --git a/man/man5/zpool-features.5 b/man/man5/zpool-features.5 index 6d74c9a78e59..fa04d6e8132e 100644 --- a/man/man5/zpool-features.5 +++ b/man/man5/zpool-features.5 @@ -432,5 +432,30 @@ set larger than 128KB, and will return to being \fBenabled\fR once all filesystems that have ever had their recordsize larger than 128KB are destroyed. .RE +.sp +.ne 2 +.na +\fB\fBlarge_dnode\fR\fR +.ad +.RS 4n +.TS +l l . +GUID org.zfsonlinux:large_dnode +READ\-ONLY COMPATIBLE no +DEPENDENCIES extensible_dataset +.TE + +The \fBlarge_dnode\fR feature allows the size of dnodes in a dataset to be +set larger than 512B. + +This feature becomes \fBactive\fR once a dataset contains an object with +a dnode larger than 512B, which occurs as a result of setting the +\fBdnodesize\fR dataset property to a value other than \fBlegacy\fR. The +feature will return to being \fBenabled\fR once all filesystems that +have ever contained a dnode larger than 512B are destroyed. Large dnodes +allow more data to be stored in the bonus buffer, thus potentially +improving performance by avoiding the use of spill blocks. +.RE + .SH "SEE ALSO" \fBzpool\fR(8) diff --git a/man/man8/zfs.8 b/man/man8/zfs.8 index 401de0378617..6921e3b4e119 100644 --- a/man/man8/zfs.8 +++ b/man/man8/zfs.8 @@ -22,7 +22,7 @@ .\" .\" Copyright (c) 2009 Sun Microsystems, Inc. All Rights Reserved. .\" Copyright 2011 Joshua M. Clulow -.\" Copyright (c) 2011, 2014 by Delphix. All rights reserved. +.\" Copyright (c) 2011, 2015 by Delphix. All rights reserved. .\" Copyright (c) 2014, Joyent, Inc. All rights reserved. .\" Copyright 2012 Nexenta Systems, Inc. All Rights Reserved. .\" Copyright (c) 2013 by Saso Kiselkov. All rights reserved. @@ -180,17 +180,27 @@ zfs \- configures ZFS file systems .LP .nf -\fBzfs\fR \fBsend\fR [\fB-eL\fR] [\fB-i \fIsnapshot\fR|\fIbookmark\fR]\fR \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR +\fBzfs\fR \fBsend\fR [\fB-Le\fR] [\fB-i \fIsnapshot\fR|\fIbookmark\fR]\fR \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR .fi .LP .nf -\fBzfs\fR \fBreceive\fR [\fB-vnFu\fR] [\fB-o origin\fR=\fIsnapshot\fR] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR +\fBzfs\fR \fBsend\fR [\fB-Penv\fR] \fB-t\fR \fIreceive_resume_token\fR .fi .LP .nf -\fBzfs\fR \fBreceive\fR [\fB-vnFu\fR] [\fB-d\fR|\fB-e\fR] [\fB-o origin\fR=\fIsnapshot\fR] \fIfilesystem\fR +\fBzfs\fR \fBreceive\fR [\fB-Fnsuv\fR] [\fB-o origin\fR=\fIsnapshot\fR] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR +.fi + +.LP +.nf +\fBzfs\fR \fBreceive\fR [\fB-Fnsuv\fR] [\fB-d\fR|\fB-e\fR] [\fB-o origin\fR=\fIsnapshot\fR] \fIfilesystem\fR +.fi + +.LP +.nf +\fBzfs\fR \fBreceive\fR \fB-A\fR \fIfilesystem\fR|\fIvolume\fR .fi .LP @@ -532,6 +542,17 @@ For cloned file systems or volumes, the snapshot from which the clone was create .sp .ne 2 .na +\fB\fBreceive_resume_token\fR\fR +.ad +.sp .6 +.RS 4n +For filesystems or volumes which have saved partially-completed state from \fBzfs receive -s\fR , this opaque token can be provided to \fBzfs send -t\fR to resume and complete the \fBzfs receive\fR. +.RE + +.sp +.ne 2 +.mk +.na \fB\fBreferenced\fR\fR .ad .sp .6 @@ -910,6 +931,35 @@ The values \fBon\fR and \fBoff\fR are equivalent to the \fBdev\fR and \fBnodev\f .sp .ne 2 .na +\fB\fBdnodesize\fR=\fBlegacy\fR | \fBauto\fR | \fB1k\fR | \fB2k\fR | \fB4k\fR | \fB8k\fR | \fB16k\fR\fR +.ad +.sp .6 +.RS 4n +Specifies a compatibility mode or literal value for the size of dnodes +in the file system. The default value is \fBlegacy\fR. Setting this +property to a value other than \fBlegacy\fR requires the +\fBlarge_dnode\fR pool feature to be enabled. +.sp +Consider setting \fBdnodesize\fR to \fBauto\fR if the dataset uses the +\fBxattr=sa\fR property setting and the workload makes heavy use of +extended attributes. This may be applicable to SELinux-enabled systems, +Lustre servers, and Samba servers, for example. Literal values are +supported for cases where the optimal size is known in advance and for +performance testing. +.sp +Leave \fBdnodesize\fR set to \fBlegacy\fR if you need to receive +a \fBzfs send\fR stream of this dataset on a pool that doesn't enable +the \fBlarge_dnode\fR feature, or if you need to import this pool on a +system that doesn't support the \fBlarge_dnode\fR feature. +.sp +This property can also be referred to by its shortened column name, +\fBdnsize\fR. +.RE + +.sp +.ne 2 +.mk +.na \fB\fBexec\fR=\fBon\fR | \fBoff\fR\fR .ad .sp .6 @@ -2770,7 +2820,7 @@ The format of the stream is committed. You will be able to receive your streams .sp .ne 2 .na -\fBzfs send\fR [\fB-eL\fR] [\fB-i\fR \fIsnapshot\fR|\fIbookmark\fR] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR +\fBzfs send\fR [\fB-Le\fR] [\fB-i\fR \fIsnapshot\fR|\fIbookmark\fR] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR .ad .sp .6 .RS 4n @@ -2780,24 +2830,6 @@ the pool must be read-only, or the filesystem must not be mounted. When the stream generated from a filesystem or volume is received, the default snapshot name will be "--head--". -.sp -.ne 2 -.na -\fB-i\fR \fIsnapshot\fR|\fIbookmark\fR -.ad -.sp .6 -.RS 4n -Generate an incremental send stream. The incremental source must be an earlier -snapshot in the destination's history. It will commonly be an earlier -snapshot in the destination's filesystem, in which case it can be -specified as the last component of the name (the \fB#\fR or \fB@\fR character -and following). -.sp -If the incremental target is a clone, the incremental source can -be the origin snapshot, or an earlier snapshot in the origin's filesystem, -or the origin's origin, etc. -.RE - .sp .ne 2 .na @@ -2830,15 +2862,39 @@ then the receiving system must have that feature enabled as well. See \fBembedded_data\fR feature. .RE +.sp +.ne 2 +.na +\fB-i\fR \fIsnapshot\fR|\fIbookmark\fR +.ad +.sp .6 +.RS 4n +Generate an incremental send stream. The incremental source must be an earlier snapshot in the destination's history. It will commonly be an earlier snapshot in the destination's filesystem, in which case it can be specified as the last component of the name (the \fB#\fR or \fB@\fR character and following). +.sp +If the incremental target is a clone, the incremental source can be the origin snapshot, or an earlier snapshot in the origin's filesystem, or the origin's origin, etc. +.RE + .RE .sp .ne 2 .na -\fB\fBzfs receive\fR [\fB-vnFu\fR] [\fB-o origin\fR=\fIsnapshot\fR] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR\fR +\fB\fBzfs send\fR [\fB-Penv\fR] \fB-t\fR \fIreceive_resume_token\fR\fR +.ad +.sp .6 +.RS 4n +Creates a send stream which resumes an interrupted receive. The \fIreceive_resume_token\fR is the value of this property on the filesystem or volume that was being received into. See the documentation for \fBzfs receive -s\fR for more details. + +.RE + +.RE +.sp +.ne 2 +.na +\fB\fBzfs receive\fR [\fB-Fnsuv\fR] [\fB-o origin\fR=\fIsnapshot\fR] \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR\fR .ad .br .na -\fB\fBzfs receive\fR [\fB-vnFu\fR] [\fB-d\fR|\fB-e\fR] [\fB-o origin\fR=\fIsnapshot\fR] \fIfilesystem\fR\fR +\fB\fBzfs receive\fR [\fB-Fnsuv\fR] [\fB-d\fR|\fB-e\fR] [\fB-o origin\fR=\fIsnapshot\fR] \fIfilesystem\fR\fR .ad .sp .6 .RS 4n @@ -2856,21 +2912,36 @@ The \fB-d\fR and \fB-e\fR options cause the file system name of the target snaps .sp .ne 2 .na -\fB\fB-d\fR\fR +\fB\fB-F\fR\fR .ad .sp .6 .RS 4n -Discard the first element of the sent snapshot's file system name, using the remaining elements to determine the name of the target file system for the new snapshot as described in the paragraph above. +Force a rollback of the file system to the most recent snapshot before performing the receive operation. If receiving an incremental replication stream (for example, one generated by \fBzfs send -R -[iI]\fR), destroy snapshots and file systems that do not exist on the sending side. .RE .sp .ne 2 +.mk .na -\fB\fB-e\fR\fR +\fB\fB-n\fR\fR .ad .sp .6 .RS 4n -Discard all but the last element of the sent snapshot's file system name, using that element to determine the name of the target file system for the new snapshot as described in the paragraph above. +Do not actually receive the stream. This can be useful in conjunction with the \fB-v\fR option to verify the name the receive operation would use. +.RE + +.sp +.ne 2 +.na +\fB\fB-s\fR\fR +.ad +.sp .6 +.RS 4n +If the receive is interrupted, save the partially received state, rather than deleting it. Interruption may be due to premature termination of the stream (e.g. due to network failure or failure of the remote system if the stream is being read over a network connection), a checksum error in the stream, termination of the \fBzfs receive\fR process, or unclean shutdown of the system. +.sp +The receive can be resumed with a stream generated by \fBzfs send -t\fR token, where the \fItoken\fR is the value of the \fBreceive_resume_token\fR property of the filesystem or volume which is received into. +.sp +To use this flag, the storage pool must have the \fBextensible_dataset\fR feature enabled. See \fBzpool-features\fR(5) for details on ZFS feature flags. .RE .sp @@ -2896,35 +2967,51 @@ Print verbose information about the stream and the time required to perform the .sp .ne 2 .na -\fB\fB-n\fR\fR +\fB\fB-d\fR\fR .ad .sp .6 .RS 4n -Do not actually receive the stream. This can be useful in conjunction with the \fB-v\fR option to verify the name the receive operation would use. +Discard the first element of the sent snapshot's file system name, using the remaining elements to determine the name of the target file system for the new snapshot as described in the paragraph above. .RE .sp .ne 2 .na -\fB\fB-o\fR \fBorigin\fR=\fIsnapshot\fR +\fB\fB-e\fR\fR .ad .sp .6 .RS 4n -Forces the stream to be received as a clone of the given snapshot. This is only valid if the stream is an incremental stream whose source is the same as the provided origin. +Discard all but the last element of the sent snapshot's file system name, using that element to determine the name of the target file system for the new snapshot as described in the paragraph above. .RE .sp .ne 2 .na -\fB\fB-F\fR\fR +\fB\fB-o\fR \fBorigin\fR=\fIsnapshot\fR .ad .sp .6 .RS 4n -Force a rollback of the file system to the most recent snapshot before performing the receive operation. If receiving an incremental replication stream (for example, one generated by \fBzfs send -R -[iI]\fR), destroy snapshots and file systems that do not exist on the sending side. +Forces the stream to be received as a clone of the given snapshot. +If the stream is a full send stream, this will create the filesystem +described by the stream as a clone of the specified snapshot. Which +snapshot was specified will not affect the success or failure of the +receive, as long as the snapshot does exist. If the stream is an +incremental send stream, all the normal verification will be performed. .RE .RE +.sp +.ne 2 +.na +\fB\fBzfs receive\fR [\fB-A\fR] \fIfilesystem\fR|\fIvolume\fR +.ad +.sp .6 +.RS 4n +Abort an interrupted \fBzfs receive \fB-s\fR\fR, deleting its saved partially received state. + +.RE + .sp .ne 2 .na @@ -2933,6 +3020,11 @@ Force a rollback of the file system to the most recent snapshot before performin .sp .6 .RS 4n Displays permissions that have been delegated on the specified filesystem or volume. See the other forms of \fBzfs allow\fR for more information. +.sp +Delegations are supported under Linux with the exception of \fBmount\fR, +\fBunmount\fR, \fBmountpoint\fR, \fBcanmount\fR, \fBrename\fR, and \fBshare\fR. +These permissions cannot be delegated because the Linux \fBmount(8)\fR command +restricts modifications of the global namespace to the root user. .RE .sp @@ -3584,9 +3676,6 @@ If you are using \fBDNS\fR for host name resolution, specify the fully qualified .LP \fBExample 17 \fRDelegating ZFS Administration Permissions on a ZFS Dataset .sp -.LP -This is not currently supported on Linux. -.sp The following example shows how to set permissions so that user \fBcindys\fR can create, destroy, mount, and take snapshots on \fBtank/cindys\fR. The permissions on \fBtank/cindys\fR are also displayed. .sp @@ -3782,6 +3871,6 @@ Invalid command line options were specified. .SH SEE ALSO .LP -\fBchmod\fR(2), \fBfsync\fR(2), \fBgzip\fR(1), \fBls\fR(1), \fBmount\fR(8), \fBopen\fR(2), \fBreaddir\fR(3), \fBssh\fR(1), \fBstat\fR(2), \fBwrite\fR(2), \fBzpool\fR(8) +\fBchmod\fR(2), \fBfsync\fR(2), \fBgzip\fR(1), \fBls\fR(1), \fBmount\fR(8), \fBopen\fR(2), \fBreaddir\fR(3), \fBssh\fR(1), \fBstat\fR(2), \fBwrite\fR(2), \fBzpool\fR(8), \fBzfs-module-parameters\fR(5) .sp On Solaris: \fBdfstab(4)\fR, \fBiscsitadm(1M)\fR, \fBmount(1M)\fR, \fBshare(1M)\fR, \fBsharemgr(1M)\fR, \fBunshare(1M)\fR diff --git a/man/man8/zinject.8 b/man/man8/zinject.8 index c21d66ac4928..90df4fb91b83 100644 --- a/man/man8/zinject.8 +++ b/man/man8/zinject.8 @@ -42,6 +42,39 @@ Cancel injection records. .TP .B "zinject \-d \fIvdev\fB \-A \fIpool\fB Force a vdev into the DEGRADED or FAULTED state. +.TP +.B "zinject -d \fIvdev\fB -D latency:lanes \fIpool\fB + +Add an artificial delay to IO requests on a particular +device, such that the requests take a minimum of 'latency' +milliseconds to complete. Each delay has an associated +number of 'lanes' which defines the number of concurrent +IO requests that can be processed. + +For example, with a single lane delay of 10 ms (-D 10:1), +the device will only be able to service a single IO request +at a time with each request taking 10 ms to complete. So, +if only a single request is submitted every 10 ms, the +average latency will be 10 ms; but if more than one request +is submitted every 10 ms, the average latency will be more +than 10 ms. + +Similarly, if a delay of 10 ms is specified to have two +lanes (-D 10:2), then the device will be able to service +two requests at a time, each with a minimum latency of +10 ms. So, if two requests are submitted every 10 ms, then +the average latency will be 10 ms; but if more than two +requests are submitted every 10 ms, the average latency +will be more than 10 ms. + +Also note, these delays are additive. So two invocations +of '-D 10:1', is roughly equivalent to a single invocation +of '-D 10:2'. This also means, one can specify multiple +lanes with differing target latencies. For example, an +invocation of '-D 10:1' followed by '-D 25:2' will +create 3 lanes on the device; one lane with a latency +of 10 ms and two lanes with a 25 ms latency. + .TP .B "zinject \-d \fIvdev\fB [\-e \fIdevice_error\fB] [\-L \fIlabel_error\fB] [\-T \fIfailure\fB] [\-F] \fIpool\fB" Force a vdev error. diff --git a/man/man8/zpool.8 b/man/man8/zpool.8 index c500eaec1ecd..80402c55ec63 100644 --- a/man/man8/zpool.8 +++ b/man/man8/zpool.8 @@ -95,7 +95,7 @@ zpool \- configures ZFS storage pools .LP .nf -\fB\fBzpool iostat\fR [\fB-T\fR \fBd\fR | \fBu\fR] [\fB-ghHLpPvy\fR] [\fB-G\fR|[\fB-lq\fR]] +\fB\fBzpool iostat\fR [\fB-T\fR \fBd\fR | \fBu\fR] [\fB-ghHLpPvy\fR] [\fB-lq\fR]|[\fB-r\fR|-\fBw\fR]] [[\fIpool\fR ...]|[\fIpool vdev\fR ...]|[\fIvdev\fR ...]] [\fIinterval\fR[\fIcount\fR]]\fR .fi @@ -1125,7 +1125,7 @@ Scripted mode. Do not display headers, and separate fields by a single tab inste \fB\fB-p\fR\fR .ad .RS 6n -Display numbers in parseable (exact) values. +Display numbers in parsable (exact) values. .RE .sp @@ -1510,7 +1510,7 @@ Scan using the default search path, the libblkid cache will not be consulted. A .sp .ne 2 .na -\fB\fBzpool iostat\fR [\fB-T\fR \fBd\fR | \fBu\fR] [\fB-ghHLpPvy\fR] [\fB-w\fR|[\fB-lq\fR]] [[\fIpool\fR ...]|[\fIpool vdev\fR ...]|[\fIvdev\fR ...]] [\fIinterval\fR[\fIcount\fR]]\fR +\fB\fBzpool iostat\fR [\fB-T\fR \fBd\fR | \fBu\fR] [\fB-ghHLpPvy\fR] [[\fB-lq\fR]|[\fB-r\fR|\fB-w\fR]] [[\fIpool\fR ...]|[\fIpool vdev\fR ...]|[\fIvdev\fR ...]] [\fIinterval\fR[\fIcount\fR]]\fR .ad .sp .6 @@ -1570,7 +1570,7 @@ Display real paths for vdevs resolving all symbolic links. This can be used to l \fB\fB-p\fR\fR .ad .RS 12n -Display numbers in parseable (exact) values. Time values are in nanoseconds. +Display numbers in parsable (exact) values. Time values are in nanoseconds. .RE .sp @@ -1582,6 +1582,19 @@ Display numbers in parseable (exact) values. Time values are in nanoseconds. Display full paths for vdevs instead of only the last component of the path. This can be used in conjunction with the \fB-L\fR flag. .RE +.sp +.ne 2 +.na +\fB\fB-r\fR\fR +.ad +.RS 12n +Print request size histograms for the leaf ZIOs. This includes histograms of +individual ZIOs ("ind") and aggregate ZIOs ("agg"). These stats can be useful +for seeing how well the ZFS IO aggregator is working. Do not confuse these +request size stats with the block layer requests; it's possible ZIOs can +be broken up before being sent to the block device. +.RE + .sp .ne 2 .na @@ -2556,4 +2569,4 @@ them on \fBzpool create\fR or \fBzpool add\fR by setting ZFS_VDEV_DEVID_OPT_OUT. .SH SEE ALSO .sp .LP -\fBzfs\fR(8), \fBzpool-features\fR(5), \fBzfs-events\fR(5) +\fBzfs\fR(8), \fBzpool-features\fR(5), \fBzfs-events\fR(5), \fBzfs-module-parameters\fR(5) diff --git a/module/Makefile.in b/module/Makefile.in index d4ddee2f429f..e4f06a6e867a 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -4,6 +4,7 @@ subdir-m += unicode subdir-m += zcommon subdir-m += zfs subdir-m += zpios +subdir-m += icp INSTALL_MOD_DIR ?= extra @@ -12,6 +13,8 @@ ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include -I@SPL@/include -I@SPL@ export ZFS_MODULE_CFLAGS +SUBDIR_TARGETS = icp + modules: @# Make the exported SPL symbols available to these modules. @# They may be in the root of SPL_OBJ when building against @@ -28,6 +31,9 @@ modules: "*** - @SPL_OBJ@/module/@SPL_SYMBOLS@\n"; \ exit 1; \ fi + list='$(SUBDIR_TARGETS)'; for targetdir in $$list; do \ + $(MAKE) -C $$targetdir; \ + done $(MAKE) -C @LINUX_OBJ@ SUBDIRS=`pwd` @KERNELMAKE_PARAMS@ CONFIG_ZFS=m $@ clean: @@ -64,8 +70,8 @@ modules_uninstall: distdir: list='$(subdir-m)'; for subdir in $$list; do \ - (find @top_srcdir@/module/$$subdir -name '*.c' -o -name '*.h' |\ - xargs /bin/cp -t $$distdir/$$subdir); \ + (cd @top_srcdir@/module && find $$subdir -name '*.c' -o -name '*.h' -o -name '*.S' |\ + xargs /bin/cp --parents -t $$distdir); \ done distclean maintainer-clean: clean diff --git a/module/avl/avl.c b/module/avl/avl.c index abf74bf7242f..86183fea01bd 100644 --- a/module/avl/avl.c +++ b/module/avl/avl.c @@ -630,7 +630,7 @@ avl_insert_here( void avl_add(avl_tree_t *tree, void *new_node) { - avl_index_t where; + avl_index_t where = 0; /* * This is unfortunate. We want to call panic() here, even for diff --git a/module/icp/Makefile.in b/module/icp/Makefile.in new file mode 100644 index 000000000000..4be03dbae50e --- /dev/null +++ b/module/icp/Makefile.in @@ -0,0 +1,82 @@ +src = @abs_top_srcdir@/module/icp +obj = @abs_builddir@ + +MODULE := icp + +TARGET_ASM_DIR = @TARGET_ASM_DIR@ + +ifeq ($(TARGET_ASM_DIR), asm-x86_64) +ASM_SOURCES := asm-x86_64/aes/aeskey.o +ASM_SOURCES += asm-x86_64/aes/aes_amd64.o +ASM_SOURCES += asm-x86_64/aes/aes_intel.o +ASM_SOURCES += asm-x86_64/modes/gcm_intel.o +ASM_SOURCES += asm-x86_64/sha1/sha1-x86_64.o +ASM_SOURCES += asm-x86_64/sha2/sha256_impl.o +endif + +ifeq ($(TARGET_ASM_DIR), asm-i386) +ASM_SOURCES := +endif + +ifeq ($(TARGET_ASM_DIR), asm-generic) +ASM_SOURCES := +endif + +EXTRA_CFLAGS = $(ZFS_MODULE_CFLAGS) @KERNELCPPFLAGS@ + +obj-$(CONFIG_ZFS) := $(MODULE).o + +ccflags-y += -I$(src)/include +asflags-y += -I$(src)/include +asflags-y += $(ZFS_MODULE_CFLAGS) + +$(MODULE)-objs += illumos-crypto.o +$(MODULE)-objs += api/kcf_cipher.o +$(MODULE)-objs += api/kcf_digest.o +$(MODULE)-objs += api/kcf_mac.o +$(MODULE)-objs += api/kcf_miscapi.o +$(MODULE)-objs += api/kcf_ctxops.o +$(MODULE)-objs += core/kcf_callprov.o +$(MODULE)-objs += core/kcf_prov_tabs.o +$(MODULE)-objs += core/kcf_sched.o +$(MODULE)-objs += core/kcf_mech_tabs.o +$(MODULE)-objs += core/kcf_prov_lib.o +$(MODULE)-objs += spi/kcf_spi.o +$(MODULE)-objs += io/aes.o +$(MODULE)-objs += io/sha1_mod.o +$(MODULE)-objs += io/sha2_mod.o +$(MODULE)-objs += os/modhash.o +$(MODULE)-objs += os/modconf.o +$(MODULE)-objs += algs/modes/cbc.o +$(MODULE)-objs += algs/modes/ccm.o +$(MODULE)-objs += algs/modes/ctr.o +$(MODULE)-objs += algs/modes/ecb.o +$(MODULE)-objs += algs/modes/gcm.o +$(MODULE)-objs += algs/modes/modes.o +$(MODULE)-objs += algs/aes/aes_impl.o +$(MODULE)-objs += algs/aes/aes_modes.o +$(MODULE)-objs += algs/sha1/sha1.o +$(MODULE)-objs += algs/sha2/sha2.o +$(MODULE)-objs += $(ASM_SOURCES) + +ICP_DIRS = \ + api \ + core \ + spi \ + io \ + os \ + algs \ + algs/aes \ + algs/modes \ + algs/sha1 \ + algs/sha2 \ + asm-x86_64 \ + asm-x86_64/aes \ + asm-x86_64/modes \ + asm-x86_64/sha1 \ + asm-x86_64/sha2 \ + asm-i386 \ + asm-generic + +all: + mkdir -p $(ICP_DIRS) diff --git a/module/icp/algs/aes/aes_impl.c b/module/icp/algs/aes/aes_impl.c new file mode 100644 index 000000000000..9c53964f04c4 --- /dev/null +++ b/module/icp/algs/aes/aes_impl.c @@ -0,0 +1,1618 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + */ + +#include +#include +#include +#include + +#ifdef __amd64 + +#ifdef _KERNEL +/* Workaround for no XMM kernel thread save/restore */ +#define KPREEMPT_DISABLE kpreempt_disable() +#define KPREEMPT_ENABLE kpreempt_enable() + +#else +#define KPREEMPT_DISABLE +#define KPREEMPT_ENABLE +#endif /* _KERNEL */ +#endif /* __amd64 */ + + +/* + * This file is derived from the file rijndael-alg-fst.c taken from the + * "optimized C code v3.0" on the "rijndael home page" + * http://www.iaik.tu-graz.ac.at/research/krypto/AES/old/~rijmen/rijndael/ + * pointed by the NIST web-site http://csrc.nist.gov/archive/aes/ + * + * The following note is from the original file: + */ + +/* + * rijndael-alg-fst.c + * + * @version 3.0 (December 2000) + * + * Optimised ANSI C code for the Rijndael cipher (now AES) + * + * @author Vincent Rijmen + * @author Antoon Bosselaers + * @author Paulo Barreto + * + * This code is hereby placed in the public domain. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ''AS IS'' AND ANY EXPRESS + * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, + * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE + * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#if defined(__amd64) + +/* These functions are used to execute amd64 instructions for AMD or Intel: */ +extern int rijndael_key_setup_enc_amd64(uint32_t rk[], + const uint32_t cipherKey[], int keyBits); +extern int rijndael_key_setup_dec_amd64(uint32_t rk[], + const uint32_t cipherKey[], int keyBits); +extern void aes_encrypt_amd64(const uint32_t rk[], int Nr, + const uint32_t pt[4], uint32_t ct[4]); +extern void aes_decrypt_amd64(const uint32_t rk[], int Nr, + const uint32_t ct[4], uint32_t pt[4]); + +/* These functions are used to execute Intel-specific AES-NI instructions: */ +extern int rijndael_key_setup_enc_intel(uint32_t rk[], + const uint32_t cipherKey[], uint64_t keyBits); +extern int rijndael_key_setup_dec_intel(uint32_t rk[], + const uint32_t cipherKey[], uint64_t keyBits); +extern void aes_encrypt_intel(const uint32_t rk[], int Nr, + const uint32_t pt[4], uint32_t ct[4]); +extern void aes_decrypt_intel(const uint32_t rk[], int Nr, + const uint32_t ct[4], uint32_t pt[4]); + +static int intel_aes_instructions_present(void); + +#define AES_ENCRYPT_IMPL(a, b, c, d, e) rijndael_encrypt(a, b, c, d, e) +#define AES_DECRYPT_IMPL(a, b, c, d, e) rijndael_decrypt(a, b, c, d, e) + +#else /* Generic C implementation */ + +#define AES_ENCRYPT_IMPL(a, b, c, d, e) rijndael_encrypt(a, b, c, d) +#define AES_DECRYPT_IMPL(a, b, c, d, e) rijndael_decrypt(a, b, c, d) +#define rijndael_key_setup_enc_raw rijndael_key_setup_enc +#endif /* __amd64 */ + +#if defined(_LITTLE_ENDIAN) && !defined(__amd64) +#define AES_BYTE_SWAP +#endif + + +#if !defined(__amd64) +/* + * Constant tables + */ + +/* + * Te0[x] = S [x].[02, 01, 01, 03]; + * Te1[x] = S [x].[03, 02, 01, 01]; + * Te2[x] = S [x].[01, 03, 02, 01]; + * Te3[x] = S [x].[01, 01, 03, 02]; + * Te4[x] = S [x].[01, 01, 01, 01]; + * + * Td0[x] = Si[x].[0e, 09, 0d, 0b]; + * Td1[x] = Si[x].[0b, 0e, 09, 0d]; + * Td2[x] = Si[x].[0d, 0b, 0e, 09]; + * Td3[x] = Si[x].[09, 0d, 0b, 0e]; + * Td4[x] = Si[x].[01, 01, 01, 01]; + */ + +/* Encrypt Sbox constants (for the substitute bytes operation) */ + +static const uint32_t Te0[256] = +{ + 0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU, + 0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U, + 0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU, + 0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU, + 0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U, + 0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU, + 0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU, + 0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU, + 0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU, + 0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU, + 0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U, + 0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU, + 0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU, + 0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U, + 0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU, + 0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU, + 0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU, + 0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU, + 0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU, + 0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U, + 0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU, + 0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU, + 0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU, + 0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU, + 0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U, + 0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U, + 0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U, + 0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U, + 0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU, + 0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U, + 0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U, + 0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU, + 0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU, + 0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U, + 0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U, + 0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U, + 0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU, + 0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U, + 0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU, + 0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U, + 0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU, + 0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U, + 0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U, + 0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU, + 0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U, + 0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U, + 0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U, + 0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U, + 0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U, + 0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U, + 0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U, + 0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U, + 0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU, + 0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U, + 0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U, + 0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U, + 0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U, + 0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U, + 0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U, + 0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU, + 0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U, + 0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U, + 0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U, + 0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU +}; + + +static const uint32_t Te1[256] = +{ + 0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU, + 0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U, + 0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU, + 0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U, + 0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU, + 0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U, + 0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU, + 0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U, + 0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U, + 0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU, + 0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U, + 0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U, + 0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U, + 0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU, + 0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U, + 0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U, + 0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU, + 0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U, + 0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U, + 0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U, + 0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU, + 0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU, + 0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U, + 0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU, + 0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU, + 0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U, + 0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU, + 0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U, + 0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU, + 0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U, + 0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U, + 0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U, + 0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU, + 0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U, + 0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU, + 0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U, + 0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU, + 0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U, + 0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U, + 0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU, + 0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU, + 0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU, + 0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U, + 0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U, + 0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU, + 0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U, + 0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU, + 0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U, + 0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU, + 0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U, + 0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU, + 0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU, + 0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U, + 0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU, + 0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U, + 0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU, + 0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U, + 0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U, + 0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U, + 0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU, + 0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU, + 0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U, + 0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU, + 0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U +}; + + +static const uint32_t Te2[256] = +{ + 0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU, + 0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U, + 0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU, + 0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U, + 0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU, + 0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U, + 0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU, + 0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U, + 0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U, + 0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU, + 0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U, + 0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U, + 0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U, + 0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU, + 0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U, + 0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U, + 0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU, + 0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U, + 0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U, + 0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U, + 0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU, + 0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU, + 0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U, + 0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU, + 0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU, + 0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U, + 0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU, + 0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U, + 0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU, + 0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U, + 0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U, + 0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U, + 0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU, + 0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U, + 0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU, + 0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U, + 0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU, + 0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U, + 0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U, + 0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU, + 0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU, + 0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU, + 0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U, + 0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U, + 0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU, + 0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U, + 0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU, + 0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U, + 0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU, + 0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U, + 0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU, + 0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU, + 0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U, + 0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU, + 0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U, + 0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU, + 0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U, + 0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U, + 0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U, + 0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU, + 0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU, + 0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U, + 0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU, + 0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U +}; + + +static const uint32_t Te3[256] = +{ + 0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U, + 0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U, + 0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U, + 0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU, + 0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU, + 0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU, + 0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U, + 0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU, + 0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU, + 0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U, + 0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U, + 0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU, + 0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU, + 0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU, + 0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU, + 0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU, + 0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U, + 0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU, + 0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU, + 0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U, + 0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U, + 0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U, + 0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U, + 0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U, + 0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU, + 0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U, + 0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU, + 0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU, + 0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U, + 0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U, + 0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U, + 0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU, + 0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U, + 0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU, + 0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU, + 0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U, + 0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U, + 0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU, + 0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U, + 0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU, + 0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U, + 0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U, + 0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U, + 0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U, + 0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU, + 0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U, + 0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU, + 0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U, + 0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU, + 0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U, + 0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU, + 0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU, + 0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU, + 0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU, + 0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U, + 0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U, + 0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U, + 0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U, + 0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U, + 0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U, + 0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU, + 0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U, + 0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU, + 0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU +}; + +static const uint32_t Te4[256] = +{ + 0x63636363U, 0x7c7c7c7cU, 0x77777777U, 0x7b7b7b7bU, + 0xf2f2f2f2U, 0x6b6b6b6bU, 0x6f6f6f6fU, 0xc5c5c5c5U, + 0x30303030U, 0x01010101U, 0x67676767U, 0x2b2b2b2bU, + 0xfefefefeU, 0xd7d7d7d7U, 0xababababU, 0x76767676U, + 0xcacacacaU, 0x82828282U, 0xc9c9c9c9U, 0x7d7d7d7dU, + 0xfafafafaU, 0x59595959U, 0x47474747U, 0xf0f0f0f0U, + 0xadadadadU, 0xd4d4d4d4U, 0xa2a2a2a2U, 0xafafafafU, + 0x9c9c9c9cU, 0xa4a4a4a4U, 0x72727272U, 0xc0c0c0c0U, + 0xb7b7b7b7U, 0xfdfdfdfdU, 0x93939393U, 0x26262626U, + 0x36363636U, 0x3f3f3f3fU, 0xf7f7f7f7U, 0xccccccccU, + 0x34343434U, 0xa5a5a5a5U, 0xe5e5e5e5U, 0xf1f1f1f1U, + 0x71717171U, 0xd8d8d8d8U, 0x31313131U, 0x15151515U, + 0x04040404U, 0xc7c7c7c7U, 0x23232323U, 0xc3c3c3c3U, + 0x18181818U, 0x96969696U, 0x05050505U, 0x9a9a9a9aU, + 0x07070707U, 0x12121212U, 0x80808080U, 0xe2e2e2e2U, + 0xebebebebU, 0x27272727U, 0xb2b2b2b2U, 0x75757575U, + 0x09090909U, 0x83838383U, 0x2c2c2c2cU, 0x1a1a1a1aU, + 0x1b1b1b1bU, 0x6e6e6e6eU, 0x5a5a5a5aU, 0xa0a0a0a0U, + 0x52525252U, 0x3b3b3b3bU, 0xd6d6d6d6U, 0xb3b3b3b3U, + 0x29292929U, 0xe3e3e3e3U, 0x2f2f2f2fU, 0x84848484U, + 0x53535353U, 0xd1d1d1d1U, 0x00000000U, 0xededededU, + 0x20202020U, 0xfcfcfcfcU, 0xb1b1b1b1U, 0x5b5b5b5bU, + 0x6a6a6a6aU, 0xcbcbcbcbU, 0xbebebebeU, 0x39393939U, + 0x4a4a4a4aU, 0x4c4c4c4cU, 0x58585858U, 0xcfcfcfcfU, + 0xd0d0d0d0U, 0xefefefefU, 0xaaaaaaaaU, 0xfbfbfbfbU, + 0x43434343U, 0x4d4d4d4dU, 0x33333333U, 0x85858585U, + 0x45454545U, 0xf9f9f9f9U, 0x02020202U, 0x7f7f7f7fU, + 0x50505050U, 0x3c3c3c3cU, 0x9f9f9f9fU, 0xa8a8a8a8U, + 0x51515151U, 0xa3a3a3a3U, 0x40404040U, 0x8f8f8f8fU, + 0x92929292U, 0x9d9d9d9dU, 0x38383838U, 0xf5f5f5f5U, + 0xbcbcbcbcU, 0xb6b6b6b6U, 0xdadadadaU, 0x21212121U, + 0x10101010U, 0xffffffffU, 0xf3f3f3f3U, 0xd2d2d2d2U, + 0xcdcdcdcdU, 0x0c0c0c0cU, 0x13131313U, 0xececececU, + 0x5f5f5f5fU, 0x97979797U, 0x44444444U, 0x17171717U, + 0xc4c4c4c4U, 0xa7a7a7a7U, 0x7e7e7e7eU, 0x3d3d3d3dU, + 0x64646464U, 0x5d5d5d5dU, 0x19191919U, 0x73737373U, + 0x60606060U, 0x81818181U, 0x4f4f4f4fU, 0xdcdcdcdcU, + 0x22222222U, 0x2a2a2a2aU, 0x90909090U, 0x88888888U, + 0x46464646U, 0xeeeeeeeeU, 0xb8b8b8b8U, 0x14141414U, + 0xdedededeU, 0x5e5e5e5eU, 0x0b0b0b0bU, 0xdbdbdbdbU, + 0xe0e0e0e0U, 0x32323232U, 0x3a3a3a3aU, 0x0a0a0a0aU, + 0x49494949U, 0x06060606U, 0x24242424U, 0x5c5c5c5cU, + 0xc2c2c2c2U, 0xd3d3d3d3U, 0xacacacacU, 0x62626262U, + 0x91919191U, 0x95959595U, 0xe4e4e4e4U, 0x79797979U, + 0xe7e7e7e7U, 0xc8c8c8c8U, 0x37373737U, 0x6d6d6d6dU, + 0x8d8d8d8dU, 0xd5d5d5d5U, 0x4e4e4e4eU, 0xa9a9a9a9U, + 0x6c6c6c6cU, 0x56565656U, 0xf4f4f4f4U, 0xeaeaeaeaU, + 0x65656565U, 0x7a7a7a7aU, 0xaeaeaeaeU, 0x08080808U, + 0xbabababaU, 0x78787878U, 0x25252525U, 0x2e2e2e2eU, + 0x1c1c1c1cU, 0xa6a6a6a6U, 0xb4b4b4b4U, 0xc6c6c6c6U, + 0xe8e8e8e8U, 0xddddddddU, 0x74747474U, 0x1f1f1f1fU, + 0x4b4b4b4bU, 0xbdbdbdbdU, 0x8b8b8b8bU, 0x8a8a8a8aU, + 0x70707070U, 0x3e3e3e3eU, 0xb5b5b5b5U, 0x66666666U, + 0x48484848U, 0x03030303U, 0xf6f6f6f6U, 0x0e0e0e0eU, + 0x61616161U, 0x35353535U, 0x57575757U, 0xb9b9b9b9U, + 0x86868686U, 0xc1c1c1c1U, 0x1d1d1d1dU, 0x9e9e9e9eU, + 0xe1e1e1e1U, 0xf8f8f8f8U, 0x98989898U, 0x11111111U, + 0x69696969U, 0xd9d9d9d9U, 0x8e8e8e8eU, 0x94949494U, + 0x9b9b9b9bU, 0x1e1e1e1eU, 0x87878787U, 0xe9e9e9e9U, + 0xcecececeU, 0x55555555U, 0x28282828U, 0xdfdfdfdfU, + 0x8c8c8c8cU, 0xa1a1a1a1U, 0x89898989U, 0x0d0d0d0dU, + 0xbfbfbfbfU, 0xe6e6e6e6U, 0x42424242U, 0x68686868U, + 0x41414141U, 0x99999999U, 0x2d2d2d2dU, 0x0f0f0f0fU, + 0xb0b0b0b0U, 0x54545454U, 0xbbbbbbbbU, 0x16161616U +}; + +/* Decrypt Sbox constants (for the substitute bytes operation) */ + +static const uint32_t Td0[256] = +{ + 0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U, + 0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U, + 0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U, + 0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU, + 0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U, + 0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U, + 0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU, + 0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U, + 0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU, + 0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U, + 0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U, + 0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U, + 0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U, + 0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU, + 0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U, + 0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU, + 0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U, + 0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU, + 0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U, + 0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U, + 0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U, + 0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU, + 0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U, + 0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU, + 0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U, + 0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU, + 0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U, + 0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU, + 0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU, + 0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U, + 0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU, + 0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U, + 0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU, + 0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U, + 0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U, + 0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U, + 0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU, + 0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U, + 0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U, + 0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU, + 0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U, + 0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U, + 0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U, + 0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U, + 0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U, + 0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU, + 0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U, + 0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U, + 0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U, + 0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U, + 0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U, + 0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU, + 0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU, + 0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU, + 0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU, + 0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U, + 0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U, + 0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU, + 0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU, + 0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U, + 0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU, + 0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U, + 0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U, + 0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U +}; + +static const uint32_t Td1[256] = +{ + 0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU, + 0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U, + 0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU, + 0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U, + 0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U, + 0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U, + 0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U, + 0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U, + 0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U, + 0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU, + 0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU, + 0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU, + 0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U, + 0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU, + 0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U, + 0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U, + 0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U, + 0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU, + 0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU, + 0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U, + 0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU, + 0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U, + 0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU, + 0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU, + 0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U, + 0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U, + 0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U, + 0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU, + 0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U, + 0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU, + 0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U, + 0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U, + 0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U, + 0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU, + 0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U, + 0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U, + 0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U, + 0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U, + 0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U, + 0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U, + 0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU, + 0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU, + 0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U, + 0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU, + 0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U, + 0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU, + 0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU, + 0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U, + 0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU, + 0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U, + 0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U, + 0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U, + 0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U, + 0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U, + 0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U, + 0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U, + 0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU, + 0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U, + 0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U, + 0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU, + 0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U, + 0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U, + 0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U, + 0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U +}; + +static const uint32_t Td2[256] = +{ + 0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U, + 0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U, + 0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U, + 0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U, + 0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU, + 0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U, + 0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U, + 0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U, + 0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U, + 0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU, + 0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U, + 0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U, + 0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU, + 0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U, + 0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U, + 0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U, + 0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U, + 0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U, + 0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U, + 0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU, + 0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U, + 0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U, + 0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U, + 0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U, + 0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U, + 0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU, + 0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU, + 0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U, + 0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU, + 0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U, + 0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU, + 0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU, + 0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU, + 0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU, + 0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U, + 0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U, + 0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U, + 0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U, + 0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U, + 0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U, + 0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U, + 0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU, + 0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU, + 0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U, + 0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U, + 0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU, + 0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU, + 0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U, + 0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U, + 0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U, + 0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U, + 0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U, + 0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U, + 0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U, + 0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU, + 0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U, + 0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U, + 0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U, + 0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U, + 0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U, + 0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U, + 0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU, + 0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U, + 0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U +}; + +static const uint32_t Td3[256] = +{ + 0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU, + 0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU, + 0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U, + 0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U, + 0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU, + 0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU, + 0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U, + 0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU, + 0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U, + 0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU, + 0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U, + 0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U, + 0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U, + 0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U, + 0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U, + 0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU, + 0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU, + 0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U, + 0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U, + 0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU, + 0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU, + 0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U, + 0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U, + 0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U, + 0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U, + 0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU, + 0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U, + 0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U, + 0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU, + 0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU, + 0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U, + 0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U, + 0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U, + 0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU, + 0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U, + 0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U, + 0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U, + 0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U, + 0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U, + 0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U, + 0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U, + 0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU, + 0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U, + 0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U, + 0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU, + 0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU, + 0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U, + 0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU, + 0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U, + 0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U, + 0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U, + 0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U, + 0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U, + 0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U, + 0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU, + 0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU, + 0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU, + 0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU, + 0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U, + 0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U, + 0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U, + 0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU, + 0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U, + 0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U +}; + +static const uint32_t Td4[256] = +{ + 0x52525252U, 0x09090909U, 0x6a6a6a6aU, 0xd5d5d5d5U, + 0x30303030U, 0x36363636U, 0xa5a5a5a5U, 0x38383838U, + 0xbfbfbfbfU, 0x40404040U, 0xa3a3a3a3U, 0x9e9e9e9eU, + 0x81818181U, 0xf3f3f3f3U, 0xd7d7d7d7U, 0xfbfbfbfbU, + 0x7c7c7c7cU, 0xe3e3e3e3U, 0x39393939U, 0x82828282U, + 0x9b9b9b9bU, 0x2f2f2f2fU, 0xffffffffU, 0x87878787U, + 0x34343434U, 0x8e8e8e8eU, 0x43434343U, 0x44444444U, + 0xc4c4c4c4U, 0xdedededeU, 0xe9e9e9e9U, 0xcbcbcbcbU, + 0x54545454U, 0x7b7b7b7bU, 0x94949494U, 0x32323232U, + 0xa6a6a6a6U, 0xc2c2c2c2U, 0x23232323U, 0x3d3d3d3dU, + 0xeeeeeeeeU, 0x4c4c4c4cU, 0x95959595U, 0x0b0b0b0bU, + 0x42424242U, 0xfafafafaU, 0xc3c3c3c3U, 0x4e4e4e4eU, + 0x08080808U, 0x2e2e2e2eU, 0xa1a1a1a1U, 0x66666666U, + 0x28282828U, 0xd9d9d9d9U, 0x24242424U, 0xb2b2b2b2U, + 0x76767676U, 0x5b5b5b5bU, 0xa2a2a2a2U, 0x49494949U, + 0x6d6d6d6dU, 0x8b8b8b8bU, 0xd1d1d1d1U, 0x25252525U, + 0x72727272U, 0xf8f8f8f8U, 0xf6f6f6f6U, 0x64646464U, + 0x86868686U, 0x68686868U, 0x98989898U, 0x16161616U, + 0xd4d4d4d4U, 0xa4a4a4a4U, 0x5c5c5c5cU, 0xccccccccU, + 0x5d5d5d5dU, 0x65656565U, 0xb6b6b6b6U, 0x92929292U, + 0x6c6c6c6cU, 0x70707070U, 0x48484848U, 0x50505050U, + 0xfdfdfdfdU, 0xededededU, 0xb9b9b9b9U, 0xdadadadaU, + 0x5e5e5e5eU, 0x15151515U, 0x46464646U, 0x57575757U, + 0xa7a7a7a7U, 0x8d8d8d8dU, 0x9d9d9d9dU, 0x84848484U, + 0x90909090U, 0xd8d8d8d8U, 0xababababU, 0x00000000U, + 0x8c8c8c8cU, 0xbcbcbcbcU, 0xd3d3d3d3U, 0x0a0a0a0aU, + 0xf7f7f7f7U, 0xe4e4e4e4U, 0x58585858U, 0x05050505U, + 0xb8b8b8b8U, 0xb3b3b3b3U, 0x45454545U, 0x06060606U, + 0xd0d0d0d0U, 0x2c2c2c2cU, 0x1e1e1e1eU, 0x8f8f8f8fU, + 0xcacacacaU, 0x3f3f3f3fU, 0x0f0f0f0fU, 0x02020202U, + 0xc1c1c1c1U, 0xafafafafU, 0xbdbdbdbdU, 0x03030303U, + 0x01010101U, 0x13131313U, 0x8a8a8a8aU, 0x6b6b6b6bU, + 0x3a3a3a3aU, 0x91919191U, 0x11111111U, 0x41414141U, + 0x4f4f4f4fU, 0x67676767U, 0xdcdcdcdcU, 0xeaeaeaeaU, + 0x97979797U, 0xf2f2f2f2U, 0xcfcfcfcfU, 0xcecececeU, + 0xf0f0f0f0U, 0xb4b4b4b4U, 0xe6e6e6e6U, 0x73737373U, + 0x96969696U, 0xacacacacU, 0x74747474U, 0x22222222U, + 0xe7e7e7e7U, 0xadadadadU, 0x35353535U, 0x85858585U, + 0xe2e2e2e2U, 0xf9f9f9f9U, 0x37373737U, 0xe8e8e8e8U, + 0x1c1c1c1cU, 0x75757575U, 0xdfdfdfdfU, 0x6e6e6e6eU, + 0x47474747U, 0xf1f1f1f1U, 0x1a1a1a1aU, 0x71717171U, + 0x1d1d1d1dU, 0x29292929U, 0xc5c5c5c5U, 0x89898989U, + 0x6f6f6f6fU, 0xb7b7b7b7U, 0x62626262U, 0x0e0e0e0eU, + 0xaaaaaaaaU, 0x18181818U, 0xbebebebeU, 0x1b1b1b1bU, + 0xfcfcfcfcU, 0x56565656U, 0x3e3e3e3eU, 0x4b4b4b4bU, + 0xc6c6c6c6U, 0xd2d2d2d2U, 0x79797979U, 0x20202020U, + 0x9a9a9a9aU, 0xdbdbdbdbU, 0xc0c0c0c0U, 0xfefefefeU, + 0x78787878U, 0xcdcdcdcdU, 0x5a5a5a5aU, 0xf4f4f4f4U, + 0x1f1f1f1fU, 0xddddddddU, 0xa8a8a8a8U, 0x33333333U, + 0x88888888U, 0x07070707U, 0xc7c7c7c7U, 0x31313131U, + 0xb1b1b1b1U, 0x12121212U, 0x10101010U, 0x59595959U, + 0x27272727U, 0x80808080U, 0xececececU, 0x5f5f5f5fU, + 0x60606060U, 0x51515151U, 0x7f7f7f7fU, 0xa9a9a9a9U, + 0x19191919U, 0xb5b5b5b5U, 0x4a4a4a4aU, 0x0d0d0d0dU, + 0x2d2d2d2dU, 0xe5e5e5e5U, 0x7a7a7a7aU, 0x9f9f9f9fU, + 0x93939393U, 0xc9c9c9c9U, 0x9c9c9c9cU, 0xefefefefU, + 0xa0a0a0a0U, 0xe0e0e0e0U, 0x3b3b3b3bU, 0x4d4d4d4dU, + 0xaeaeaeaeU, 0x2a2a2a2aU, 0xf5f5f5f5U, 0xb0b0b0b0U, + 0xc8c8c8c8U, 0xebebebebU, 0xbbbbbbbbU, 0x3c3c3c3cU, + 0x83838383U, 0x53535353U, 0x99999999U, 0x61616161U, + 0x17171717U, 0x2b2b2b2bU, 0x04040404U, 0x7e7e7e7eU, + 0xbabababaU, 0x77777777U, 0xd6d6d6d6U, 0x26262626U, + 0xe1e1e1e1U, 0x69696969U, 0x14141414U, 0x63636363U, + 0x55555555U, 0x21212121U, 0x0c0c0c0cU, 0x7d7d7d7dU +}; + +/* Rcon is Round Constant; used for encryption key expansion */ +static const uint32_t rcon[RC_LENGTH] = +{ + /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */ + 0x01000000, 0x02000000, 0x04000000, 0x08000000, + 0x10000000, 0x20000000, 0x40000000, 0x80000000, + 0x1B000000, 0x36000000 +}; + + +/* + * Expand the cipher key into the encryption key schedule. + * + * Return the number of rounds for the given cipher key size. + * The size of the key schedule depends on the number of rounds + * (which can be computed from the size of the key), i.e. 4*(Nr + 1). + * + * Parameters: + * rk AES key schedule 32-bit array to be initialized + * cipherKey User key + * keyBits AES key size (128, 192, or 256 bits) + */ +static int +rijndael_key_setup_enc_raw(uint32_t rk[], const uint32_t cipherKey[], + int keyBits) +{ + int i = 0; + uint32_t temp; + + rk[0] = cipherKey[0]; + rk[1] = cipherKey[1]; + rk[2] = cipherKey[2]; + rk[3] = cipherKey[3]; + + if (keyBits == 128) { + for (;;) { + temp = rk[3]; + rk[4] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[temp & 0xff] & 0x0000ff00) ^ + (Te4[temp >> 24] & 0x000000ff) ^ + rcon[i]; + rk[5] = rk[1] ^ rk[4]; + rk[6] = rk[2] ^ rk[5]; + rk[7] = rk[3] ^ rk[6]; + + if (++i == 10) { + return (10); + } + rk += 4; + } + } + + rk[4] = cipherKey[4]; + rk[5] = cipherKey[5]; + + if (keyBits == 192) { + for (;;) { + temp = rk[5]; + rk[6] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[temp & 0xff] & 0x0000ff00) ^ + (Te4[temp >> 24] & 0x000000ff) ^ + rcon[i]; + rk[7] = rk[1] ^ rk[6]; + rk[8] = rk[2] ^ rk[7]; + rk[9] = rk[3] ^ rk[8]; + + if (++i == 8) { + return (12); + } + + rk[10] = rk[4] ^ rk[9]; + rk[11] = rk[5] ^ rk[10]; + rk += 6; + } + } + + rk[6] = cipherKey[6]; + rk[7] = cipherKey[7]; + + if (keyBits == 256) { + for (;;) { + temp = rk[7]; + rk[8] = rk[0] ^ + (Te4[(temp >> 16) & 0xff] & 0xff000000) ^ + (Te4[(temp >> 8) & 0xff] & 0x00ff0000) ^ + (Te4[temp & 0xff] & 0x0000ff00) ^ + (Te4[temp >> 24] & 0x000000ff) ^ + rcon[i]; + rk[9] = rk[1] ^ rk[8]; + rk[10] = rk[2] ^ rk[9]; + rk[11] = rk[3] ^ rk[10]; + + if (++i == 7) { + return (14); + } + temp = rk[11]; + rk[12] = rk[4] ^ + (Te4[temp >> 24] & 0xff000000) ^ + (Te4[(temp >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(temp >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[temp & 0xff] & 0x000000ff); + rk[13] = rk[5] ^ rk[12]; + rk[14] = rk[6] ^ rk[13]; + rk[15] = rk[7] ^ rk[14]; + + rk += 8; + } + } + + return (0); +} +#endif /* !__amd64 */ + +#if defined(__amd64) + +/* + * Expand the 32-bit AES cipher key array into the encryption and decryption + * key schedules. + * + * Parameters: + * key AES key schedule to be initialized + * keyarr32 User key + * keyBits AES key size (128, 192, or 256 bits) + */ +static void +aes_setupkeys(aes_key_t *key, const uint32_t *keyarr32, int keybits) +{ + if (intel_aes_instructions_present()) { + key->flags = INTEL_AES_NI_CAPABLE; + KPREEMPT_DISABLE; + key->nr = rijndael_key_setup_enc_intel(&(key->encr_ks.ks32[0]), + keyarr32, keybits); + key->nr = rijndael_key_setup_dec_intel(&(key->decr_ks.ks32[0]), + keyarr32, keybits); + KPREEMPT_ENABLE; + } else { + key->flags = 0; + key->nr = rijndael_key_setup_enc_amd64(&(key->encr_ks.ks32[0]), + keyarr32, keybits); + key->nr = rijndael_key_setup_dec_amd64(&(key->decr_ks.ks32[0]), + keyarr32, keybits); + } + + key->type = AES_32BIT_KS; +} + +/* + * Encrypt one block of data. The block is assumed to be an array + * of four uint32_t values, so copy for alignment (and byte-order + * reversal for little endian systems might be necessary on the + * input and output byte streams. + * The size of the key schedule depends on the number of rounds + * (which can be computed from the size of the key), i.e. 4*(Nr + 1). + * + * Parameters: + * rk Key schedule, of aes_ks_t (60 32-bit integers) + * Nr Number of rounds + * pt Input block (plain text) + * ct Output block (crypto text). Can overlap with pt + * flags Indicates whether we're on Intel AES-NI-capable hardware + */ +static void +rijndael_encrypt(const uint32_t rk[], int Nr, const uint32_t pt[4], + uint32_t ct[4], int flags) { + if (flags & INTEL_AES_NI_CAPABLE) { + KPREEMPT_DISABLE; + aes_encrypt_intel(rk, Nr, pt, ct); + KPREEMPT_ENABLE; + } else { + aes_encrypt_amd64(rk, Nr, pt, ct); + } +} + +/* + * Decrypt one block of data. The block is assumed to be an array + * of four uint32_t values, so copy for alignment (and byte-order + * reversal for little endian systems might be necessary on the + * input and output byte streams. + * The size of the key schedule depends on the number of rounds + * (which can be computed from the size of the key), i.e. 4*(Nr + 1). + * + * Parameters: + * rk Key schedule, of aes_ks_t (60 32-bit integers) + * Nr Number of rounds + * ct Input block (crypto text) + * pt Output block (plain text). Can overlap with pt + * flags Indicates whether we're on Intel AES-NI-capable hardware + */ +static void +rijndael_decrypt(const uint32_t rk[], int Nr, const uint32_t ct[4], + uint32_t pt[4], int flags) { + if (flags & INTEL_AES_NI_CAPABLE) { + KPREEMPT_DISABLE; + aes_decrypt_intel(rk, Nr, ct, pt); + KPREEMPT_ENABLE; + } else { + aes_decrypt_amd64(rk, Nr, ct, pt); + } +} + + +#else /* generic C implementation */ + +/* + * Expand the cipher key into the decryption key schedule. + * Return the number of rounds for the given cipher key size. + * The size of the key schedule depends on the number of rounds + * (which can be computed from the size of the key), i.e. 4*(Nr + 1). + * + * Parameters: + * rk AES key schedule 32-bit array to be initialized + * cipherKey User key + * keyBits AES key size (128, 192, or 256 bits) + */ +static int +rijndael_key_setup_dec(uint32_t rk[], const uint32_t cipherKey[], int keyBits) +{ + int Nr, i, j; + uint32_t temp; + + /* expand the cipher key: */ + Nr = rijndael_key_setup_enc_raw(rk, cipherKey, keyBits); + + /* invert the order of the round keys: */ + for (i = 0, j = 4 * Nr; i < j; i += 4, j -= 4) { + temp = rk[i]; + rk[i] = rk[j]; + rk[j] = temp; + temp = rk[i + 1]; + rk[i + 1] = rk[j + 1]; + rk[j + 1] = temp; + temp = rk[i + 2]; + rk[i + 2] = rk[j + 2]; + rk[j + 2] = temp; + temp = rk[i + 3]; + rk[i + 3] = rk[j + 3]; + rk[j + 3] = temp; + } + + /* + * apply the inverse MixColumn transform to all + * round keys but the first and the last: + */ + for (i = 1; i < Nr; i++) { + rk += 4; + rk[0] = Td0[Te4[rk[0] >> 24] & 0xff] ^ + Td1[Te4[(rk[0] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[0] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[rk[0] & 0xff] & 0xff]; + rk[1] = Td0[Te4[rk[1] >> 24] & 0xff] ^ + Td1[Te4[(rk[1] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[1] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[rk[1] & 0xff] & 0xff]; + rk[2] = Td0[Te4[rk[2] >> 24] & 0xff] ^ + Td1[Te4[(rk[2] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[2] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[rk[2] & 0xff] & 0xff]; + rk[3] = Td0[Te4[rk[3] >> 24] & 0xff] ^ + Td1[Te4[(rk[3] >> 16) & 0xff] & 0xff] ^ + Td2[Te4[(rk[3] >> 8) & 0xff] & 0xff] ^ + Td3[Te4[rk[3] & 0xff] & 0xff]; + } + + return (Nr); +} + + +/* + * Expand the 32-bit AES cipher key array into the encryption and decryption + * key schedules. + * + * Parameters: + * key AES key schedule to be initialized + * keyarr32 User key + * keyBits AES key size (128, 192, or 256 bits) + */ +static void +aes_setupkeys(aes_key_t *key, const uint32_t *keyarr32, int keybits) +{ + key->nr = rijndael_key_setup_enc(&(key->encr_ks.ks32[0]), keyarr32, + keybits); + key->nr = rijndael_key_setup_dec(&(key->decr_ks.ks32[0]), keyarr32, + keybits); + key->type = AES_32BIT_KS; +} + + +/* + * Encrypt one block of data. The block is assumed to be an array + * of four uint32_t values, so copy for alignment (and byte-order + * reversal for little endian systems might be necessary on the + * input and output byte streams. + * The size of the key schedule depends on the number of rounds + * (which can be computed from the size of the key), i.e. 4*(Nr + 1). + * + * Parameters: + * rk Key schedule, of aes_ks_t (60 32-bit integers) + * Nr Number of rounds + * pt Input block (plain text) + * ct Output block (crypto text). Can overlap with pt + */ +static void +rijndael_encrypt(const uint32_t rk[], int Nr, const uint32_t pt[4], + uint32_t ct[4]) +{ + uint32_t s0, s1, s2, s3, t0, t1, t2, t3; + int r; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + + s0 = pt[0] ^ rk[0]; + s1 = pt[1] ^ rk[1]; + s2 = pt[2] ^ rk[2]; + s3 = pt[3] ^ rk[3]; + + /* + * Nr - 1 full rounds: + */ + + r = Nr >> 1; + + for (;;) { + t0 = Te0[s0 >> 24] ^ + Te1[(s1 >> 16) & 0xff] ^ + Te2[(s2 >> 8) & 0xff] ^ + Te3[s3 & 0xff] ^ + rk[4]; + + t1 = Te0[s1 >> 24] ^ + Te1[(s2 >> 16) & 0xff] ^ + Te2[(s3 >> 8) & 0xff] ^ + Te3[s0 & 0xff] ^ + rk[5]; + + t2 = Te0[s2 >> 24] ^ + Te1[(s3 >> 16) & 0xff] ^ + Te2[(s0 >> 8) & 0xff] ^ + Te3[s1 & 0xff] ^ + rk[6]; + + t3 = Te0[s3 >> 24] ^ + Te1[(s0 >> 16) & 0xff] ^ + Te2[(s1 >> 8) & 0xff] ^ + Te3[s2 & 0xff] ^ + rk[7]; + + rk += 8; + + if (--r == 0) { + break; + } + + s0 = Te0[t0 >> 24] ^ + Te1[(t1 >> 16) & 0xff] ^ + Te2[(t2 >> 8) & 0xff] ^ + Te3[t3 & 0xff] ^ + rk[0]; + + s1 = Te0[t1 >> 24] ^ + Te1[(t2 >> 16) & 0xff] ^ + Te2[(t3 >> 8) & 0xff] ^ + Te3[t0 & 0xff] ^ + rk[1]; + + s2 = Te0[t2 >> 24] ^ + Te1[(t3 >> 16) & 0xff] ^ + Te2[(t0 >> 8) & 0xff] ^ + Te3[t1 & 0xff] ^ + rk[2]; + + s3 = Te0[t3 >> 24] ^ + Te1[(t0 >> 16) & 0xff] ^ + Te2[(t1 >> 8) & 0xff] ^ + Te3[t2 & 0xff] ^ + rk[3]; + } + + /* + * apply last round and + * map cipher state to byte array block: + */ + + s0 = (Te4[(t0 >> 24)] & 0xff000000) ^ + (Te4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[t3 & 0xff] & 0x000000ff) ^ + rk[0]; + ct[0] = s0; + + s1 = (Te4[(t1 >> 24)] & 0xff000000) ^ + (Te4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[t0 & 0xff] & 0x000000ff) ^ + rk[1]; + ct[1] = s1; + + s2 = (Te4[(t2 >> 24)] & 0xff000000) ^ + (Te4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[t1 & 0xff] & 0x000000ff) ^ + rk[2]; + ct[2] = s2; + + s3 = (Te4[(t3 >> 24)] & 0xff000000) ^ + (Te4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Te4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ + (Te4[t2 & 0xff] & 0x000000ff) ^ + rk[3]; + ct[3] = s3; +} + + +/* + * Decrypt one block of data. The block is assumed to be an array + * of four uint32_t values, so copy for alignment (and byte-order + * reversal for little endian systems might be necessary on the + * input and output byte streams. + * The size of the key schedule depends on the number of rounds + * (which can be computed from the size of the key), i.e. 4*(Nr + 1). + * + * Parameters: + * rk Key schedule, of aes_ks_t (60 32-bit integers) + * Nr Number of rounds + * ct Input block (crypto text) + * pt Output block (plain text). Can overlap with pt + */ +static void +rijndael_decrypt(const uint32_t rk[], int Nr, const uint32_t ct[4], + uint32_t pt[4]) +{ + uint32_t s0, s1, s2, s3, t0, t1, t2, t3; + int r; + + /* + * map byte array block to cipher state + * and add initial round key: + */ + s0 = ct[0] ^ rk[0]; + s1 = ct[1] ^ rk[1]; + s2 = ct[2] ^ rk[2]; + s3 = ct[3] ^ rk[3]; + + /* + * Nr - 1 full rounds: + */ + + r = Nr >> 1; + + for (;;) { + t0 = Td0[s0 >> 24] ^ + Td1[(s3 >> 16) & 0xff] ^ + Td2[(s2 >> 8) & 0xff] ^ + Td3[s1 & 0xff] ^ + rk[4]; + + t1 = Td0[s1 >> 24] ^ + Td1[(s0 >> 16) & 0xff] ^ + Td2[(s3 >> 8) & 0xff] ^ + Td3[s2 & 0xff] ^ + rk[5]; + + t2 = Td0[s2 >> 24] ^ + Td1[(s1 >> 16) & 0xff] ^ + Td2[(s0 >> 8) & 0xff] ^ + Td3[s3 & 0xff] ^ + rk[6]; + + t3 = Td0[s3 >> 24] ^ + Td1[(s2 >> 16) & 0xff] ^ + Td2[(s1 >> 8) & 0xff] ^ + Td3[s0 & 0xff] ^ + rk[7]; + + rk += 8; + + if (--r == 0) { + break; + } + + s0 = Td0[t0 >> 24] ^ + Td1[(t3 >> 16) & 0xff] ^ + Td2[(t2 >> 8) & 0xff] ^ + Td3[t1 & 0xff] ^ + rk[0]; + + s1 = Td0[t1 >> 24] ^ + Td1[(t0 >> 16) & 0xff] ^ + Td2[(t3 >> 8) & 0xff] ^ + Td3[t2 & 0xff] ^ + rk[1]; + + s2 = Td0[t2 >> 24] ^ + Td1[(t1 >> 16) & 0xff] ^ + Td2[(t0 >> 8) & 0xff] ^ + Td3[t3 & 0xff] ^ + rk[2]; + + s3 = Td0[t3 >> 24] ^ + Td1[(t2 >> 16) & 0xff] ^ + Td2[(t1 >> 8) & 0xff] ^ + Td3[t0 & 0xff] ^ + rk[3]; + } + + /* + * apply last round and + * map cipher state to byte array block: + */ + + s0 = (Td4[t0 >> 24] & 0xff000000) ^ + (Td4[(t3 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t2 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[t1 & 0xff] & 0x000000ff) ^ + rk[0]; + pt[0] = s0; + + s1 = (Td4[t1 >> 24] & 0xff000000) ^ + (Td4[(t0 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t3 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[t2 & 0xff] & 0x000000ff) ^ + rk[1]; + pt[1] = s1; + + s2 = (Td4[t2 >> 24] & 0xff000000) ^ + (Td4[(t1 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t0 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[t3 & 0xff] & 0x000000ff) ^ + rk[2]; + pt[2] = s2; + + s3 = (Td4[t3 >> 24] & 0xff000000) ^ + (Td4[(t2 >> 16) & 0xff] & 0x00ff0000) ^ + (Td4[(t1 >> 8) & 0xff] & 0x0000ff00) ^ + (Td4[t0 & 0xff] & 0x000000ff) ^ + rk[3]; + pt[3] = s3; +} +#endif /* __amd64 */ + + +/* + * Initialize AES encryption and decryption key schedules. + * + * Parameters: + * cipherKey User key + * keyBits AES key size (128, 192, or 256 bits) + * keysched AES key schedule to be initialized, of type aes_key_t. + * Allocated by aes_alloc_keysched(). + */ +void +aes_init_keysched(const uint8_t *cipherKey, uint_t keyBits, void *keysched) +{ + aes_key_t *newbie = keysched; + uint_t keysize, i, j; + union { + uint64_t ka64[4]; + uint32_t ka32[8]; + } keyarr; + + switch (keyBits) { + case 128: + newbie->nr = 10; + break; + + case 192: + newbie->nr = 12; + break; + + case 256: + newbie->nr = 14; + break; + + default: + /* should never get here */ + return; + } + keysize = CRYPTO_BITS2BYTES(keyBits); + + /* + * For _LITTLE_ENDIAN machines (except AMD64), reverse every + * 4 bytes in the key. On _BIG_ENDIAN and AMD64, copy the key + * without reversing bytes. + * For AMD64, do not byte swap for aes_setupkeys(). + * + * SPARCv8/v9 uses a key schedule array with 64-bit elements. + * X86/AMD64 uses a key schedule array with 32-bit elements. + */ +#ifndef AES_BYTE_SWAP + if (IS_P2ALIGNED(cipherKey, sizeof (uint64_t))) { + for (i = 0, j = 0; j < keysize; i++, j += 8) { + /* LINTED: pointer alignment */ + keyarr.ka64[i] = *((uint64_t *)&cipherKey[j]); + } + } else { + bcopy(cipherKey, keyarr.ka32, keysize); + } + +#else /* byte swap */ + for (i = 0, j = 0; j < keysize; i++, j += 4) { + keyarr.ka32[i] = htonl(*(uint32_t *)(void *)&cipherKey[j]); + } +#endif + + aes_setupkeys(newbie, keyarr.ka32, keyBits); +} + + +/* + * Encrypt one block using AES. + * Align if needed and (for x86 32-bit only) byte-swap. + * + * Parameters: + * ks Key schedule, of type aes_key_t + * pt Input block (plain text) + * ct Output block (crypto text). Can overlap with pt + */ +int +aes_encrypt_block(const void *ks, const uint8_t *pt, uint8_t *ct) +{ + aes_key_t *ksch = (aes_key_t *)ks; + +#ifndef AES_BYTE_SWAP + if (IS_P2ALIGNED2(pt, ct, sizeof (uint32_t))) { + /* LINTED: pointer alignment */ + AES_ENCRYPT_IMPL(&ksch->encr_ks.ks32[0], ksch->nr, + /* LINTED: pointer alignment */ + (uint32_t *)pt, (uint32_t *)ct, ksch->flags); + } else { +#endif + uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)]; + + /* Copy input block into buffer */ +#ifndef AES_BYTE_SWAP + bcopy(pt, &buffer, AES_BLOCK_LEN); + +#else /* byte swap */ + buffer[0] = htonl(*(uint32_t *)(void *)&pt[0]); + buffer[1] = htonl(*(uint32_t *)(void *)&pt[4]); + buffer[2] = htonl(*(uint32_t *)(void *)&pt[8]); + buffer[3] = htonl(*(uint32_t *)(void *)&pt[12]); +#endif + + AES_ENCRYPT_IMPL(&ksch->encr_ks.ks32[0], ksch->nr, + buffer, buffer, ksch->flags); + + /* Copy result from buffer to output block */ +#ifndef AES_BYTE_SWAP + bcopy(&buffer, ct, AES_BLOCK_LEN); + } + +#else /* byte swap */ + *(uint32_t *)(void *)&ct[0] = htonl(buffer[0]); + *(uint32_t *)(void *)&ct[4] = htonl(buffer[1]); + *(uint32_t *)(void *)&ct[8] = htonl(buffer[2]); + *(uint32_t *)(void *)&ct[12] = htonl(buffer[3]); +#endif + return (CRYPTO_SUCCESS); +} + + +/* + * Decrypt one block using AES. + * Align and byte-swap if needed. + * + * Parameters: + * ks Key schedule, of type aes_key_t + * ct Input block (crypto text) + * pt Output block (plain text). Can overlap with pt + */ +int +aes_decrypt_block(const void *ks, const uint8_t *ct, uint8_t *pt) +{ + aes_key_t *ksch = (aes_key_t *)ks; + +#ifndef AES_BYTE_SWAP + if (IS_P2ALIGNED2(ct, pt, sizeof (uint32_t))) { + /* LINTED: pointer alignment */ + AES_DECRYPT_IMPL(&ksch->decr_ks.ks32[0], ksch->nr, + /* LINTED: pointer alignment */ + (uint32_t *)ct, (uint32_t *)pt, ksch->flags); + } else { +#endif + uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)]; + + /* Copy input block into buffer */ +#ifndef AES_BYTE_SWAP + bcopy(ct, &buffer, AES_BLOCK_LEN); + +#else /* byte swap */ + buffer[0] = htonl(*(uint32_t *)(void *)&ct[0]); + buffer[1] = htonl(*(uint32_t *)(void *)&ct[4]); + buffer[2] = htonl(*(uint32_t *)(void *)&ct[8]); + buffer[3] = htonl(*(uint32_t *)(void *)&ct[12]); +#endif + + AES_DECRYPT_IMPL(&ksch->decr_ks.ks32[0], ksch->nr, + buffer, buffer, ksch->flags); + + /* Copy result from buffer to output block */ +#ifndef AES_BYTE_SWAP + bcopy(&buffer, pt, AES_BLOCK_LEN); + } + +#else /* byte swap */ + *(uint32_t *)(void *)&pt[0] = htonl(buffer[0]); + *(uint32_t *)(void *)&pt[4] = htonl(buffer[1]); + *(uint32_t *)(void *)&pt[8] = htonl(buffer[2]); + *(uint32_t *)(void *)&pt[12] = htonl(buffer[3]); +#endif + + return (CRYPTO_SUCCESS); +} + + +/* + * Allocate key schedule for AES. + * + * Return the pointer and set size to the number of bytes allocated. + * Memory allocated must be freed by the caller when done. + * + * Parameters: + * size Size of key schedule allocated, in bytes + * kmflag Flag passed to kmem_alloc(9F); ignored in userland. + */ +/* ARGSUSED */ +void * +aes_alloc_keysched(size_t *size, int kmflag) +{ + aes_key_t *keysched; + + keysched = (aes_key_t *)kmem_alloc(sizeof (aes_key_t), kmflag); + if (keysched != NULL) { + *size = sizeof (aes_key_t); + return (keysched); + } + return (NULL); +} + + +#ifdef __amd64 + +#define INTEL_AESNI_FLAG (1 << 25) + +/* + * Return 1 if executing on Intel with AES-NI instructions, + * otherwise 0 (i.e., Intel without AES-NI or AMD64). + * Cache the result, as the CPU can't change. + */ +static int +intel_aes_instructions_present(void) +{ + static int cached_result = -1; + unsigned eax, ebx, ecx, edx; + unsigned func, subfunc; + + if (cached_result == -1) { /* first time */ + /* check for an intel cpu */ + func = 0; + subfunc = 0; + + __asm__ __volatile__( + "cpuid" + : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) + : "a"(func), "c"(subfunc)); + + if (memcmp((char *) (&ebx), "Genu", 4) == 0 && + memcmp((char *) (&edx), "ineI", 4) == 0 && + memcmp((char *) (&ecx), "ntel", 4) == 0) { + + func = 1; + subfunc = 0; + + /* check for aes-ni instruction set */ + __asm__ __volatile__( + "cpuid" + : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) + : "a"(func), "c"(subfunc)); + + cached_result = !!(ecx & INTEL_AESNI_FLAG); + } else { + cached_result = 0; + } + } + + return (cached_result); +} + +#endif /* __amd64 */ diff --git a/module/icp/algs/aes/aes_modes.c b/module/icp/algs/aes/aes_modes.c new file mode 100644 index 000000000000..9e4b498fffcb --- /dev/null +++ b/module/icp/algs/aes/aes_modes.c @@ -0,0 +1,135 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include + +/* Copy a 16-byte AES block from "in" to "out" */ +void +aes_copy_block(uint8_t *in, uint8_t *out) +{ + if (IS_P2ALIGNED2(in, out, sizeof (uint32_t))) { + /* LINTED: pointer alignment */ + *(uint32_t *)&out[0] = *(uint32_t *)&in[0]; + /* LINTED: pointer alignment */ + *(uint32_t *)&out[4] = *(uint32_t *)&in[4]; + /* LINTED: pointer alignment */ + *(uint32_t *)&out[8] = *(uint32_t *)&in[8]; + /* LINTED: pointer alignment */ + *(uint32_t *)&out[12] = *(uint32_t *)&in[12]; + } else { + AES_COPY_BLOCK(in, out); + } +} + + +/* XOR a 16-byte AES block of data into dst */ +void +aes_xor_block(uint8_t *data, uint8_t *dst) +{ + if (IS_P2ALIGNED2(dst, data, sizeof (uint32_t))) { + /* LINTED: pointer alignment */ + *(uint32_t *)&dst[0] ^= *(uint32_t *)&data[0]; + /* LINTED: pointer alignment */ + *(uint32_t *)&dst[4] ^= *(uint32_t *)&data[4]; + /* LINTED: pointer alignment */ + *(uint32_t *)&dst[8] ^= *(uint32_t *)&data[8]; + /* LINTED: pointer alignment */ + *(uint32_t *)&dst[12] ^= *(uint32_t *)&data[12]; + } else { + AES_XOR_BLOCK(data, dst); + } +} + + +/* + * Encrypt multiple blocks of data according to mode. + */ +int +aes_encrypt_contiguous_blocks(void *ctx, char *data, size_t length, + crypto_data_t *out) +{ + aes_ctx_t *aes_ctx = ctx; + int rv; + + if (aes_ctx->ac_flags & CTR_MODE) { + rv = ctr_mode_contiguous_blocks(ctx, data, length, out, + AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); + } else if (aes_ctx->ac_flags & CCM_MODE) { + rv = ccm_mode_encrypt_contiguous_blocks(ctx, data, length, + out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + rv = gcm_mode_encrypt_contiguous_blocks(ctx, data, length, + out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + } else if (aes_ctx->ac_flags & CBC_MODE) { + rv = cbc_encrypt_contiguous_blocks(ctx, + data, length, out, AES_BLOCK_LEN, aes_encrypt_block, + aes_copy_block, aes_xor_block); + } else { + rv = ecb_cipher_contiguous_blocks(ctx, data, length, out, + AES_BLOCK_LEN, aes_encrypt_block); + } + return (rv); +} + + +/* + * Decrypt multiple blocks of data according to mode. + */ +int +aes_decrypt_contiguous_blocks(void *ctx, char *data, size_t length, + crypto_data_t *out) +{ + aes_ctx_t *aes_ctx = ctx; + int rv; + + if (aes_ctx->ac_flags & CTR_MODE) { + rv = ctr_mode_contiguous_blocks(ctx, data, length, out, + AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); + if (rv == CRYPTO_DATA_LEN_RANGE) + rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; + } else if (aes_ctx->ac_flags & CCM_MODE) { + rv = ccm_mode_decrypt_contiguous_blocks(ctx, data, length, + out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + rv = gcm_mode_decrypt_contiguous_blocks(ctx, data, length, + out, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + } else if (aes_ctx->ac_flags & CBC_MODE) { + rv = cbc_decrypt_contiguous_blocks(ctx, data, length, out, + AES_BLOCK_LEN, aes_decrypt_block, aes_copy_block, + aes_xor_block); + } else { + rv = ecb_cipher_contiguous_blocks(ctx, data, length, out, + AES_BLOCK_LEN, aes_decrypt_block); + if (rv == CRYPTO_DATA_LEN_RANGE) + rv = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; + } + return (rv); +} diff --git a/module/icp/algs/modes/cbc.c b/module/icp/algs/modes/cbc.c new file mode 100644 index 000000000000..2cc94ec72656 --- /dev/null +++ b/module/icp/algs/modes/cbc.c @@ -0,0 +1,305 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include + +/* + * Algorithm independent CBC functions. + */ +int +cbc_encrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length, + crypto_data_t *out, size_t block_size, + int (*encrypt)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + size_t remainder = length; + size_t need = 0; + uint8_t *datap = (uint8_t *)data; + uint8_t *blockp; + uint8_t *lastp; + void *iov_or_mp; + offset_t offset; + uint8_t *out_data_1; + uint8_t *out_data_2; + size_t out_data_1_len; + + if (length + ctx->cbc_remainder_len < block_size) { + /* accumulate bytes here and return */ + bcopy(datap, + (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len, + length); + ctx->cbc_remainder_len += length; + ctx->cbc_copy_to = datap; + return (CRYPTO_SUCCESS); + } + + lastp = (uint8_t *)ctx->cbc_iv; + if (out != NULL) + crypto_init_ptrs(out, &iov_or_mp, &offset); + + do { + /* Unprocessed data from last call. */ + if (ctx->cbc_remainder_len > 0) { + need = block_size - ctx->cbc_remainder_len; + + if (need > remainder) + return (CRYPTO_DATA_LEN_RANGE); + + bcopy(datap, &((uint8_t *)ctx->cbc_remainder) + [ctx->cbc_remainder_len], need); + + blockp = (uint8_t *)ctx->cbc_remainder; + } else { + blockp = datap; + } + + if (out == NULL) { + /* + * XOR the previous cipher block or IV with the + * current clear block. + */ + xor_block(lastp, blockp); + encrypt(ctx->cbc_keysched, blockp, blockp); + + ctx->cbc_lastp = blockp; + lastp = blockp; + + if (ctx->cbc_remainder_len > 0) { + bcopy(blockp, ctx->cbc_copy_to, + ctx->cbc_remainder_len); + bcopy(blockp + ctx->cbc_remainder_len, datap, + need); + } + } else { + /* + * XOR the previous cipher block or IV with the + * current clear block. + */ + xor_block(blockp, lastp); + encrypt(ctx->cbc_keysched, lastp, lastp); + crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, + &out_data_1_len, &out_data_2, block_size); + + /* copy block to where it belongs */ + if (out_data_1_len == block_size) { + copy_block(lastp, out_data_1); + } else { + bcopy(lastp, out_data_1, out_data_1_len); + if (out_data_2 != NULL) { + bcopy(lastp + out_data_1_len, + out_data_2, + block_size - out_data_1_len); + } + } + /* update offset */ + out->cd_offset += block_size; + } + + /* Update pointer to next block of data to be processed. */ + if (ctx->cbc_remainder_len != 0) { + datap += need; + ctx->cbc_remainder_len = 0; + } else { + datap += block_size; + } + + remainder = (size_t)&data[length] - (size_t)datap; + + /* Incomplete last block. */ + if (remainder > 0 && remainder < block_size) { + bcopy(datap, ctx->cbc_remainder, remainder); + ctx->cbc_remainder_len = remainder; + ctx->cbc_copy_to = datap; + goto out; + } + ctx->cbc_copy_to = NULL; + + } while (remainder > 0); + +out: + /* + * Save the last encrypted block in the context. + */ + if (ctx->cbc_lastp != NULL) { + copy_block((uint8_t *)ctx->cbc_lastp, (uint8_t *)ctx->cbc_iv); + ctx->cbc_lastp = (uint8_t *)ctx->cbc_iv; + } + + return (CRYPTO_SUCCESS); +} + +#define OTHER(a, ctx) \ + (((a) == (ctx)->cbc_lastblock) ? (ctx)->cbc_iv : (ctx)->cbc_lastblock) + +/* ARGSUSED */ +int +cbc_decrypt_contiguous_blocks(cbc_ctx_t *ctx, char *data, size_t length, + crypto_data_t *out, size_t block_size, + int (*decrypt)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + size_t remainder = length; + size_t need = 0; + uint8_t *datap = (uint8_t *)data; + uint8_t *blockp; + uint8_t *lastp; + void *iov_or_mp; + offset_t offset; + uint8_t *out_data_1; + uint8_t *out_data_2; + size_t out_data_1_len; + + if (length + ctx->cbc_remainder_len < block_size) { + /* accumulate bytes here and return */ + bcopy(datap, + (uint8_t *)ctx->cbc_remainder + ctx->cbc_remainder_len, + length); + ctx->cbc_remainder_len += length; + ctx->cbc_copy_to = datap; + return (CRYPTO_SUCCESS); + } + + lastp = ctx->cbc_lastp; + if (out != NULL) + crypto_init_ptrs(out, &iov_or_mp, &offset); + + do { + /* Unprocessed data from last call. */ + if (ctx->cbc_remainder_len > 0) { + need = block_size - ctx->cbc_remainder_len; + + if (need > remainder) + return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); + + bcopy(datap, &((uint8_t *)ctx->cbc_remainder) + [ctx->cbc_remainder_len], need); + + blockp = (uint8_t *)ctx->cbc_remainder; + } else { + blockp = datap; + } + + /* LINTED: pointer alignment */ + copy_block(blockp, (uint8_t *)OTHER((uint64_t *)lastp, ctx)); + + if (out != NULL) { + decrypt(ctx->cbc_keysched, blockp, + (uint8_t *)ctx->cbc_remainder); + blockp = (uint8_t *)ctx->cbc_remainder; + } else { + decrypt(ctx->cbc_keysched, blockp, blockp); + } + + /* + * XOR the previous cipher block or IV with the + * currently decrypted block. + */ + xor_block(lastp, blockp); + + /* LINTED: pointer alignment */ + lastp = (uint8_t *)OTHER((uint64_t *)lastp, ctx); + + if (out != NULL) { + crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, + &out_data_1_len, &out_data_2, block_size); + + bcopy(blockp, out_data_1, out_data_1_len); + if (out_data_2 != NULL) { + bcopy(blockp + out_data_1_len, out_data_2, + block_size - out_data_1_len); + } + + /* update offset */ + out->cd_offset += block_size; + + } else if (ctx->cbc_remainder_len > 0) { + /* copy temporary block to where it belongs */ + bcopy(blockp, ctx->cbc_copy_to, ctx->cbc_remainder_len); + bcopy(blockp + ctx->cbc_remainder_len, datap, need); + } + + /* Update pointer to next block of data to be processed. */ + if (ctx->cbc_remainder_len != 0) { + datap += need; + ctx->cbc_remainder_len = 0; + } else { + datap += block_size; + } + + remainder = (size_t)&data[length] - (size_t)datap; + + /* Incomplete last block. */ + if (remainder > 0 && remainder < block_size) { + bcopy(datap, ctx->cbc_remainder, remainder); + ctx->cbc_remainder_len = remainder; + ctx->cbc_lastp = lastp; + ctx->cbc_copy_to = datap; + return (CRYPTO_SUCCESS); + } + ctx->cbc_copy_to = NULL; + + } while (remainder > 0); + + ctx->cbc_lastp = lastp; + return (CRYPTO_SUCCESS); +} + +int +cbc_init_ctx(cbc_ctx_t *cbc_ctx, char *param, size_t param_len, + size_t block_size, void (*copy_block)(uint8_t *, uint64_t *)) +{ + /* + * Copy IV into context. + * + * If cm_param == NULL then the IV comes from the + * cd_miscdata field in the crypto_data structure. + */ + if (param != NULL) { + ASSERT(param_len == block_size); + copy_block((uchar_t *)param, cbc_ctx->cbc_iv); + } + + cbc_ctx->cbc_lastp = (uint8_t *)&cbc_ctx->cbc_iv[0]; + cbc_ctx->cbc_flags |= CBC_MODE; + return (CRYPTO_SUCCESS); +} + +/* ARGSUSED */ +void * +cbc_alloc_ctx(int kmflag) +{ + cbc_ctx_t *cbc_ctx; + + if ((cbc_ctx = kmem_zalloc(sizeof (cbc_ctx_t), kmflag)) == NULL) + return (NULL); + + cbc_ctx->cbc_flags = CBC_MODE; + return (cbc_ctx); +} diff --git a/module/icp/algs/modes/ccm.c b/module/icp/algs/modes/ccm.c new file mode 100644 index 000000000000..22aeb0a6aa47 --- /dev/null +++ b/module/icp/algs/modes/ccm.c @@ -0,0 +1,920 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include + +#if defined(__i386) || defined(__amd64) +#include +#define UNALIGNED_POINTERS_PERMITTED +#endif + +/* + * Encrypt multiple blocks of data in CCM mode. Decrypt for CCM mode + * is done in another function. + */ +int +ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length, + crypto_data_t *out, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + size_t remainder = length; + size_t need = 0; + uint8_t *datap = (uint8_t *)data; + uint8_t *blockp; + uint8_t *lastp; + void *iov_or_mp; + offset_t offset; + uint8_t *out_data_1; + uint8_t *out_data_2; + size_t out_data_1_len; + uint64_t counter; + uint8_t *mac_buf; + + if (length + ctx->ccm_remainder_len < block_size) { + /* accumulate bytes here and return */ + bcopy(datap, + (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len, + length); + ctx->ccm_remainder_len += length; + ctx->ccm_copy_to = datap; + return (CRYPTO_SUCCESS); + } + + lastp = (uint8_t *)ctx->ccm_cb; + if (out != NULL) + crypto_init_ptrs(out, &iov_or_mp, &offset); + + mac_buf = (uint8_t *)ctx->ccm_mac_buf; + + do { + /* Unprocessed data from last call. */ + if (ctx->ccm_remainder_len > 0) { + need = block_size - ctx->ccm_remainder_len; + + if (need > remainder) + return (CRYPTO_DATA_LEN_RANGE); + + bcopy(datap, &((uint8_t *)ctx->ccm_remainder) + [ctx->ccm_remainder_len], need); + + blockp = (uint8_t *)ctx->ccm_remainder; + } else { + blockp = datap; + } + + /* + * do CBC MAC + * + * XOR the previous cipher block current clear block. + * mac_buf always contain previous cipher block. + */ + xor_block(blockp, mac_buf); + encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf); + + /* ccm_cb is the counter block */ + encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, + (uint8_t *)ctx->ccm_tmp); + + lastp = (uint8_t *)ctx->ccm_tmp; + + /* + * Increment counter. Counter bits are confined + * to the bottom 64 bits of the counter block. + */ +#ifdef _LITTLE_ENDIAN + counter = ntohll(ctx->ccm_cb[1] & ctx->ccm_counter_mask); + counter = htonll(counter + 1); +#else + counter = ctx->ccm_cb[1] & ctx->ccm_counter_mask; + counter++; +#endif /* _LITTLE_ENDIAN */ + counter &= ctx->ccm_counter_mask; + ctx->ccm_cb[1] = + (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter; + + /* + * XOR encrypted counter block with the current clear block. + */ + xor_block(blockp, lastp); + + ctx->ccm_processed_data_len += block_size; + + if (out == NULL) { + if (ctx->ccm_remainder_len > 0) { + bcopy(blockp, ctx->ccm_copy_to, + ctx->ccm_remainder_len); + bcopy(blockp + ctx->ccm_remainder_len, datap, + need); + } + } else { + crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, + &out_data_1_len, &out_data_2, block_size); + + /* copy block to where it belongs */ + if (out_data_1_len == block_size) { + copy_block(lastp, out_data_1); + } else { + bcopy(lastp, out_data_1, out_data_1_len); + if (out_data_2 != NULL) { + bcopy(lastp + out_data_1_len, + out_data_2, + block_size - out_data_1_len); + } + } + /* update offset */ + out->cd_offset += block_size; + } + + /* Update pointer to next block of data to be processed. */ + if (ctx->ccm_remainder_len != 0) { + datap += need; + ctx->ccm_remainder_len = 0; + } else { + datap += block_size; + } + + remainder = (size_t)&data[length] - (size_t)datap; + + /* Incomplete last block. */ + if (remainder > 0 && remainder < block_size) { + bcopy(datap, ctx->ccm_remainder, remainder); + ctx->ccm_remainder_len = remainder; + ctx->ccm_copy_to = datap; + goto out; + } + ctx->ccm_copy_to = NULL; + + } while (remainder > 0); + +out: + return (CRYPTO_SUCCESS); +} + +void +calculate_ccm_mac(ccm_ctx_t *ctx, uint8_t *ccm_mac, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)) +{ + uint64_t counter; + uint8_t *counterp, *mac_buf; + int i; + + mac_buf = (uint8_t *)ctx->ccm_mac_buf; + + /* first counter block start with index 0 */ + counter = 0; + ctx->ccm_cb[1] = (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter; + + counterp = (uint8_t *)ctx->ccm_tmp; + encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp); + + /* calculate XOR of MAC with first counter block */ + for (i = 0; i < ctx->ccm_mac_len; i++) { + ccm_mac[i] = mac_buf[i] ^ counterp[i]; + } +} + +/* ARGSUSED */ +int +ccm_encrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + uint8_t *lastp, *mac_buf, *ccm_mac_p, *macp = NULL; + void *iov_or_mp; + offset_t offset; + uint8_t *out_data_1; + uint8_t *out_data_2; + size_t out_data_1_len; + int i; + + if (out->cd_length < (ctx->ccm_remainder_len + ctx->ccm_mac_len)) { + return (CRYPTO_DATA_LEN_RANGE); + } + + /* + * When we get here, the number of bytes of payload processed + * plus whatever data remains, if any, + * should be the same as the number of bytes that's being + * passed in the argument during init time. + */ + if ((ctx->ccm_processed_data_len + ctx->ccm_remainder_len) + != (ctx->ccm_data_len)) { + return (CRYPTO_DATA_LEN_RANGE); + } + + mac_buf = (uint8_t *)ctx->ccm_mac_buf; + + if (ctx->ccm_remainder_len > 0) { + + /* ccm_mac_input_buf is not used for encryption */ + macp = (uint8_t *)ctx->ccm_mac_input_buf; + bzero(macp, block_size); + + /* copy remainder to temporary buffer */ + bcopy(ctx->ccm_remainder, macp, ctx->ccm_remainder_len); + + /* calculate the CBC MAC */ + xor_block(macp, mac_buf); + encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf); + + /* calculate the counter mode */ + lastp = (uint8_t *)ctx->ccm_tmp; + encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, lastp); + + /* XOR with counter block */ + for (i = 0; i < ctx->ccm_remainder_len; i++) { + macp[i] ^= lastp[i]; + } + ctx->ccm_processed_data_len += ctx->ccm_remainder_len; + } + + /* Calculate the CCM MAC */ + ccm_mac_p = (uint8_t *)ctx->ccm_tmp; + calculate_ccm_mac(ctx, ccm_mac_p, encrypt_block); + + crypto_init_ptrs(out, &iov_or_mp, &offset); + crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, + &out_data_1_len, &out_data_2, + ctx->ccm_remainder_len + ctx->ccm_mac_len); + + if (ctx->ccm_remainder_len > 0) { + + /* copy temporary block to where it belongs */ + if (out_data_2 == NULL) { + /* everything will fit in out_data_1 */ + bcopy(macp, out_data_1, ctx->ccm_remainder_len); + bcopy(ccm_mac_p, out_data_1 + ctx->ccm_remainder_len, + ctx->ccm_mac_len); + } else { + + if (out_data_1_len < ctx->ccm_remainder_len) { + + size_t data_2_len_used; + + bcopy(macp, out_data_1, out_data_1_len); + + data_2_len_used = ctx->ccm_remainder_len + - out_data_1_len; + + bcopy((uint8_t *)macp + out_data_1_len, + out_data_2, data_2_len_used); + bcopy(ccm_mac_p, out_data_2 + data_2_len_used, + ctx->ccm_mac_len); + } else { + bcopy(macp, out_data_1, out_data_1_len); + if (out_data_1_len == ctx->ccm_remainder_len) { + /* mac will be in out_data_2 */ + bcopy(ccm_mac_p, out_data_2, + ctx->ccm_mac_len); + } else { + size_t len_not_used = out_data_1_len - + ctx->ccm_remainder_len; + /* + * part of mac in will be in + * out_data_1, part of the mac will be + * in out_data_2 + */ + bcopy(ccm_mac_p, + out_data_1 + ctx->ccm_remainder_len, + len_not_used); + bcopy(ccm_mac_p + len_not_used, + out_data_2, + ctx->ccm_mac_len - len_not_used); + + } + } + } + } else { + /* copy block to where it belongs */ + bcopy(ccm_mac_p, out_data_1, out_data_1_len); + if (out_data_2 != NULL) { + bcopy(ccm_mac_p + out_data_1_len, out_data_2, + block_size - out_data_1_len); + } + } + out->cd_offset += ctx->ccm_remainder_len + ctx->ccm_mac_len; + ctx->ccm_remainder_len = 0; + return (CRYPTO_SUCCESS); +} + +/* + * This will only deal with decrypting the last block of the input that + * might not be a multiple of block length. + */ +void +ccm_decrypt_incomplete_block(ccm_ctx_t *ctx, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)) +{ + uint8_t *datap, *outp, *counterp; + int i; + + datap = (uint8_t *)ctx->ccm_remainder; + outp = &((ctx->ccm_pt_buf)[ctx->ccm_processed_data_len]); + + counterp = (uint8_t *)ctx->ccm_tmp; + encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, counterp); + + /* XOR with counter block */ + for (i = 0; i < ctx->ccm_remainder_len; i++) { + outp[i] = datap[i] ^ counterp[i]; + } +} + +/* + * This will decrypt the cipher text. However, the plaintext won't be + * returned to the caller. It will be returned when decrypt_final() is + * called if the MAC matches + */ +/* ARGSUSED */ +int +ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *ctx, char *data, size_t length, + crypto_data_t *out, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + size_t remainder = length; + size_t need = 0; + uint8_t *datap = (uint8_t *)data; + uint8_t *blockp; + uint8_t *cbp; + uint64_t counter; + size_t pt_len, total_decrypted_len, mac_len, pm_len, pd_len; + uint8_t *resultp; + + + pm_len = ctx->ccm_processed_mac_len; + + if (pm_len > 0) { + uint8_t *tmp; + /* + * all ciphertext has been processed, just waiting for + * part of the value of the mac + */ + if ((pm_len + length) > ctx->ccm_mac_len) { + return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); + } + tmp = (uint8_t *)ctx->ccm_mac_input_buf; + + bcopy(datap, tmp + pm_len, length); + + ctx->ccm_processed_mac_len += length; + return (CRYPTO_SUCCESS); + } + + /* + * If we decrypt the given data, what total amount of data would + * have been decrypted? + */ + pd_len = ctx->ccm_processed_data_len; + total_decrypted_len = pd_len + length + ctx->ccm_remainder_len; + + if (total_decrypted_len > + (ctx->ccm_data_len + ctx->ccm_mac_len)) { + return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); + } + + pt_len = ctx->ccm_data_len; + + if (total_decrypted_len > pt_len) { + /* + * part of the input will be the MAC, need to isolate that + * to be dealt with later. The left-over data in + * ccm_remainder_len from last time will not be part of the + * MAC. Otherwise, it would have already been taken out + * when this call is made last time. + */ + size_t pt_part = pt_len - pd_len - ctx->ccm_remainder_len; + + mac_len = length - pt_part; + + ctx->ccm_processed_mac_len = mac_len; + bcopy(data + pt_part, ctx->ccm_mac_input_buf, mac_len); + + if (pt_part + ctx->ccm_remainder_len < block_size) { + /* + * since this is last of the ciphertext, will + * just decrypt with it here + */ + bcopy(datap, &((uint8_t *)ctx->ccm_remainder) + [ctx->ccm_remainder_len], pt_part); + ctx->ccm_remainder_len += pt_part; + ccm_decrypt_incomplete_block(ctx, encrypt_block); + ctx->ccm_processed_data_len += ctx->ccm_remainder_len; + ctx->ccm_remainder_len = 0; + return (CRYPTO_SUCCESS); + } else { + /* let rest of the code handle this */ + length = pt_part; + } + } else if (length + ctx->ccm_remainder_len < block_size) { + /* accumulate bytes here and return */ + bcopy(datap, + (uint8_t *)ctx->ccm_remainder + ctx->ccm_remainder_len, + length); + ctx->ccm_remainder_len += length; + ctx->ccm_copy_to = datap; + return (CRYPTO_SUCCESS); + } + + do { + /* Unprocessed data from last call. */ + if (ctx->ccm_remainder_len > 0) { + need = block_size - ctx->ccm_remainder_len; + + if (need > remainder) + return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); + + bcopy(datap, &((uint8_t *)ctx->ccm_remainder) + [ctx->ccm_remainder_len], need); + + blockp = (uint8_t *)ctx->ccm_remainder; + } else { + blockp = datap; + } + + /* Calculate the counter mode, ccm_cb is the counter block */ + cbp = (uint8_t *)ctx->ccm_tmp; + encrypt_block(ctx->ccm_keysched, (uint8_t *)ctx->ccm_cb, cbp); + + /* + * Increment counter. + * Counter bits are confined to the bottom 64 bits + */ +#ifdef _LITTLE_ENDIAN + counter = ntohll(ctx->ccm_cb[1] & ctx->ccm_counter_mask); + counter = htonll(counter + 1); +#else + counter = ctx->ccm_cb[1] & ctx->ccm_counter_mask; + counter++; +#endif /* _LITTLE_ENDIAN */ + counter &= ctx->ccm_counter_mask; + ctx->ccm_cb[1] = + (ctx->ccm_cb[1] & ~(ctx->ccm_counter_mask)) | counter; + + /* XOR with the ciphertext */ + xor_block(blockp, cbp); + + /* Copy the plaintext to the "holding buffer" */ + resultp = (uint8_t *)ctx->ccm_pt_buf + + ctx->ccm_processed_data_len; + copy_block(cbp, resultp); + + ctx->ccm_processed_data_len += block_size; + + ctx->ccm_lastp = blockp; + + /* Update pointer to next block of data to be processed. */ + if (ctx->ccm_remainder_len != 0) { + datap += need; + ctx->ccm_remainder_len = 0; + } else { + datap += block_size; + } + + remainder = (size_t)&data[length] - (size_t)datap; + + /* Incomplete last block */ + if (remainder > 0 && remainder < block_size) { + bcopy(datap, ctx->ccm_remainder, remainder); + ctx->ccm_remainder_len = remainder; + ctx->ccm_copy_to = datap; + if (ctx->ccm_processed_mac_len > 0) { + /* + * not expecting anymore ciphertext, just + * compute plaintext for the remaining input + */ + ccm_decrypt_incomplete_block(ctx, + encrypt_block); + ctx->ccm_processed_data_len += remainder; + ctx->ccm_remainder_len = 0; + } + goto out; + } + ctx->ccm_copy_to = NULL; + + } while (remainder > 0); + +out: + return (CRYPTO_SUCCESS); +} + +int +ccm_decrypt_final(ccm_ctx_t *ctx, crypto_data_t *out, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + size_t mac_remain, pt_len; + uint8_t *pt, *mac_buf, *macp, *ccm_mac_p; + int rv; + + pt_len = ctx->ccm_data_len; + + /* Make sure output buffer can fit all of the plaintext */ + if (out->cd_length < pt_len) { + return (CRYPTO_DATA_LEN_RANGE); + } + + pt = ctx->ccm_pt_buf; + mac_remain = ctx->ccm_processed_data_len; + mac_buf = (uint8_t *)ctx->ccm_mac_buf; + + macp = (uint8_t *)ctx->ccm_tmp; + + while (mac_remain > 0) { + + if (mac_remain < block_size) { + bzero(macp, block_size); + bcopy(pt, macp, mac_remain); + mac_remain = 0; + } else { + copy_block(pt, macp); + mac_remain -= block_size; + pt += block_size; + } + + /* calculate the CBC MAC */ + xor_block(macp, mac_buf); + encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf); + } + + /* Calculate the CCM MAC */ + ccm_mac_p = (uint8_t *)ctx->ccm_tmp; + calculate_ccm_mac((ccm_ctx_t *)ctx, ccm_mac_p, encrypt_block); + + /* compare the input CCM MAC value with what we calculated */ + if (bcmp(ctx->ccm_mac_input_buf, ccm_mac_p, ctx->ccm_mac_len)) { + /* They don't match */ + return (CRYPTO_INVALID_MAC); + } else { + rv = crypto_put_output_data(ctx->ccm_pt_buf, out, pt_len); + if (rv != CRYPTO_SUCCESS) + return (rv); + out->cd_offset += pt_len; + } + return (CRYPTO_SUCCESS); +} + +int +ccm_validate_args(CK_AES_CCM_PARAMS *ccm_param, boolean_t is_encrypt_init) +{ + size_t macSize, nonceSize; + uint8_t q; + uint64_t maxValue; + + /* + * Check the length of the MAC. The only valid + * lengths for the MAC are: 4, 6, 8, 10, 12, 14, 16 + */ + macSize = ccm_param->ulMACSize; + if ((macSize < 4) || (macSize > 16) || ((macSize % 2) != 0)) { + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + + /* Check the nonce length. Valid values are 7, 8, 9, 10, 11, 12, 13 */ + nonceSize = ccm_param->ulNonceSize; + if ((nonceSize < 7) || (nonceSize > 13)) { + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + + /* q is the length of the field storing the length, in bytes */ + q = (uint8_t)((15 - nonceSize) & 0xFF); + + + /* + * If it is decrypt, need to make sure size of ciphertext is at least + * bigger than MAC len + */ + if ((!is_encrypt_init) && (ccm_param->ulDataSize < macSize)) { + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + + /* + * Check to make sure the length of the payload is within the + * range of values allowed by q + */ + if (q < 8) { + maxValue = (1ULL << (q * 8)) - 1; + } else { + maxValue = ULONG_MAX; + } + + if (ccm_param->ulDataSize > maxValue) { + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + return (CRYPTO_SUCCESS); +} + +/* + * Format the first block used in CBC-MAC (B0) and the initial counter + * block based on formatting functions and counter generation functions + * specified in RFC 3610 and NIST publication 800-38C, appendix A + * + * b0 is the first block used in CBC-MAC + * cb0 is the first counter block + * + * It's assumed that the arguments b0 and cb0 are preallocated AES blocks + * + */ +static void +ccm_format_initial_blocks(uchar_t *nonce, ulong_t nonceSize, + ulong_t authDataSize, uint8_t *b0, ccm_ctx_t *aes_ctx) +{ + uint64_t payloadSize; + uint8_t t, q, have_adata = 0; + size_t limit; + int i, j, k; + uint64_t mask = 0; + uint8_t *cb; + + q = (uint8_t)((15 - nonceSize) & 0xFF); + t = (uint8_t)((aes_ctx->ccm_mac_len) & 0xFF); + + /* Construct the first octet of b0 */ + if (authDataSize > 0) { + have_adata = 1; + } + b0[0] = (have_adata << 6) | (((t - 2) / 2) << 3) | (q - 1); + + /* copy the nonce value into b0 */ + bcopy(nonce, &(b0[1]), nonceSize); + + /* store the length of the payload into b0 */ + bzero(&(b0[1+nonceSize]), q); + + payloadSize = aes_ctx->ccm_data_len; + limit = 8 < q ? 8 : q; + + for (i = 0, j = 0, k = 15; i < limit; i++, j += 8, k--) { + b0[k] = (uint8_t)((payloadSize >> j) & 0xFF); + } + + /* format the counter block */ + + cb = (uint8_t *)aes_ctx->ccm_cb; + + cb[0] = 0x07 & (q-1); /* first byte */ + + /* copy the nonce value into the counter block */ + bcopy(nonce, &(cb[1]), nonceSize); + + bzero(&(cb[1+nonceSize]), q); + + /* Create the mask for the counter field based on the size of nonce */ + q <<= 3; + while (q-- > 0) { + mask |= (1ULL << q); + } + +#ifdef _LITTLE_ENDIAN + mask = htonll(mask); +#endif + aes_ctx->ccm_counter_mask = mask; + + /* + * During calculation, we start using counter block 1, we will + * set it up right here. + * We can just set the last byte to have the value 1, because + * even with the biggest nonce of 13, the last byte of the + * counter block will be used for the counter value. + */ + cb[15] = 0x01; +} + +/* + * Encode the length of the associated data as + * specified in RFC 3610 and NIST publication 800-38C, appendix A + */ +static void +encode_adata_len(ulong_t auth_data_len, uint8_t *encoded, size_t *encoded_len) +{ +#ifdef UNALIGNED_POINTERS_PERMITTED + uint32_t *lencoded_ptr; +#ifdef _LP64 + uint64_t *llencoded_ptr; +#endif +#endif /* UNALIGNED_POINTERS_PERMITTED */ + + if (auth_data_len < ((1ULL<<16) - (1ULL<<8))) { + /* 0 < a < (2^16-2^8) */ + *encoded_len = 2; + encoded[0] = (auth_data_len & 0xff00) >> 8; + encoded[1] = auth_data_len & 0xff; + + } else if ((auth_data_len >= ((1ULL<<16) - (1ULL<<8))) && + (auth_data_len < (1ULL << 31))) { + /* (2^16-2^8) <= a < 2^32 */ + *encoded_len = 6; + encoded[0] = 0xff; + encoded[1] = 0xfe; +#ifdef UNALIGNED_POINTERS_PERMITTED + lencoded_ptr = (uint32_t *)&encoded[2]; + *lencoded_ptr = htonl(auth_data_len); +#else + encoded[2] = (auth_data_len & 0xff000000) >> 24; + encoded[3] = (auth_data_len & 0xff0000) >> 16; + encoded[4] = (auth_data_len & 0xff00) >> 8; + encoded[5] = auth_data_len & 0xff; +#endif /* UNALIGNED_POINTERS_PERMITTED */ + +#ifdef _LP64 + } else { + /* 2^32 <= a < 2^64 */ + *encoded_len = 10; + encoded[0] = 0xff; + encoded[1] = 0xff; +#ifdef UNALIGNED_POINTERS_PERMITTED + llencoded_ptr = (uint64_t *)&encoded[2]; + *llencoded_ptr = htonl(auth_data_len); +#else + encoded[2] = (auth_data_len & 0xff00000000000000) >> 56; + encoded[3] = (auth_data_len & 0xff000000000000) >> 48; + encoded[4] = (auth_data_len & 0xff0000000000) >> 40; + encoded[5] = (auth_data_len & 0xff00000000) >> 32; + encoded[6] = (auth_data_len & 0xff000000) >> 24; + encoded[7] = (auth_data_len & 0xff0000) >> 16; + encoded[8] = (auth_data_len & 0xff00) >> 8; + encoded[9] = auth_data_len & 0xff; +#endif /* UNALIGNED_POINTERS_PERMITTED */ +#endif /* _LP64 */ + } +} + +/* + * The following function should be call at encrypt or decrypt init time + * for AES CCM mode. + */ +int +ccm_init(ccm_ctx_t *ctx, unsigned char *nonce, size_t nonce_len, + unsigned char *auth_data, size_t auth_data_len, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + uint8_t *mac_buf, *datap, *ivp, *authp; + size_t remainder, processed; + uint8_t encoded_a[10]; /* max encoded auth data length is 10 octets */ + size_t encoded_a_len = 0; + + mac_buf = (uint8_t *)&(ctx->ccm_mac_buf); + + /* + * Format the 1st block for CBC-MAC and construct the + * 1st counter block. + * + * aes_ctx->ccm_iv is used for storing the counter block + * mac_buf will store b0 at this time. + */ + ccm_format_initial_blocks(nonce, nonce_len, + auth_data_len, mac_buf, ctx); + + /* The IV for CBC MAC for AES CCM mode is always zero */ + ivp = (uint8_t *)ctx->ccm_tmp; + bzero(ivp, block_size); + + xor_block(ivp, mac_buf); + + /* encrypt the nonce */ + encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf); + + /* take care of the associated data, if any */ + if (auth_data_len == 0) { + return (CRYPTO_SUCCESS); + } + + encode_adata_len(auth_data_len, encoded_a, &encoded_a_len); + + remainder = auth_data_len; + + /* 1st block: it contains encoded associated data, and some data */ + authp = (uint8_t *)ctx->ccm_tmp; + bzero(authp, block_size); + bcopy(encoded_a, authp, encoded_a_len); + processed = block_size - encoded_a_len; + if (processed > auth_data_len) { + /* in case auth_data is very small */ + processed = auth_data_len; + } + bcopy(auth_data, authp+encoded_a_len, processed); + /* xor with previous buffer */ + xor_block(authp, mac_buf); + encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf); + remainder -= processed; + if (remainder == 0) { + /* a small amount of associated data, it's all done now */ + return (CRYPTO_SUCCESS); + } + + do { + if (remainder < block_size) { + /* + * There's not a block full of data, pad rest of + * buffer with zero + */ + bzero(authp, block_size); + bcopy(&(auth_data[processed]), authp, remainder); + datap = (uint8_t *)authp; + remainder = 0; + } else { + datap = (uint8_t *)(&(auth_data[processed])); + processed += block_size; + remainder -= block_size; + } + + xor_block(datap, mac_buf); + encrypt_block(ctx->ccm_keysched, mac_buf, mac_buf); + + } while (remainder > 0); + + return (CRYPTO_SUCCESS); +} + +int +ccm_init_ctx(ccm_ctx_t *ccm_ctx, char *param, int kmflag, + boolean_t is_encrypt_init, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + int rv; + CK_AES_CCM_PARAMS *ccm_param; + + if (param != NULL) { + ccm_param = (CK_AES_CCM_PARAMS *)param; + + if ((rv = ccm_validate_args(ccm_param, + is_encrypt_init)) != 0) { + return (rv); + } + + ccm_ctx->ccm_mac_len = ccm_param->ulMACSize; + if (is_encrypt_init) { + ccm_ctx->ccm_data_len = ccm_param->ulDataSize; + } else { + ccm_ctx->ccm_data_len = + ccm_param->ulDataSize - ccm_ctx->ccm_mac_len; + ccm_ctx->ccm_processed_mac_len = 0; + } + ccm_ctx->ccm_processed_data_len = 0; + + ccm_ctx->ccm_flags |= CCM_MODE; + } else { + rv = CRYPTO_MECHANISM_PARAM_INVALID; + goto out; + } + + if (ccm_init(ccm_ctx, ccm_param->nonce, ccm_param->ulNonceSize, + ccm_param->authData, ccm_param->ulAuthDataSize, block_size, + encrypt_block, xor_block) != 0) { + rv = CRYPTO_MECHANISM_PARAM_INVALID; + goto out; + } + if (!is_encrypt_init) { + /* allocate buffer for storing decrypted plaintext */ + ccm_ctx->ccm_pt_buf = vmem_alloc(ccm_ctx->ccm_data_len, + kmflag); + if (ccm_ctx->ccm_pt_buf == NULL) { + rv = CRYPTO_HOST_MEMORY; + } + } +out: + return (rv); +} + +void * +ccm_alloc_ctx(int kmflag) +{ + ccm_ctx_t *ccm_ctx; + + if ((ccm_ctx = kmem_zalloc(sizeof (ccm_ctx_t), kmflag)) == NULL) + return (NULL); + + ccm_ctx->ccm_flags = CCM_MODE; + return (ccm_ctx); +} diff --git a/module/icp/algs/modes/ctr.c b/module/icp/algs/modes/ctr.c new file mode 100644 index 000000000000..77ba28dddfc0 --- /dev/null +++ b/module/icp/algs/modes/ctr.c @@ -0,0 +1,238 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include + +/* + * Encrypt and decrypt multiple blocks of data in counter mode. + */ +int +ctr_mode_contiguous_blocks(ctr_ctx_t *ctx, char *data, size_t length, + crypto_data_t *out, size_t block_size, + int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + size_t remainder = length; + size_t need = 0; + uint8_t *datap = (uint8_t *)data; + uint8_t *blockp; + uint8_t *lastp; + void *iov_or_mp; + offset_t offset; + uint8_t *out_data_1; + uint8_t *out_data_2; + size_t out_data_1_len; + uint64_t lower_counter, upper_counter; + + if (length + ctx->ctr_remainder_len < block_size) { + /* accumulate bytes here and return */ + bcopy(datap, + (uint8_t *)ctx->ctr_remainder + ctx->ctr_remainder_len, + length); + ctx->ctr_remainder_len += length; + ctx->ctr_copy_to = datap; + return (CRYPTO_SUCCESS); + } + + lastp = (uint8_t *)ctx->ctr_cb; + if (out != NULL) + crypto_init_ptrs(out, &iov_or_mp, &offset); + + do { + /* Unprocessed data from last call. */ + if (ctx->ctr_remainder_len > 0) { + need = block_size - ctx->ctr_remainder_len; + + if (need > remainder) + return (CRYPTO_DATA_LEN_RANGE); + + bcopy(datap, &((uint8_t *)ctx->ctr_remainder) + [ctx->ctr_remainder_len], need); + + blockp = (uint8_t *)ctx->ctr_remainder; + } else { + blockp = datap; + } + + /* ctr_cb is the counter block */ + cipher(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb, + (uint8_t *)ctx->ctr_tmp); + + lastp = (uint8_t *)ctx->ctr_tmp; + + /* + * Increment Counter. + */ + lower_counter = ntohll(ctx->ctr_cb[1] & ctx->ctr_lower_mask); + lower_counter = htonll(lower_counter + 1); + lower_counter &= ctx->ctr_lower_mask; + ctx->ctr_cb[1] = (ctx->ctr_cb[1] & ~(ctx->ctr_lower_mask)) | + lower_counter; + + /* wrap around */ + if (lower_counter == 0) { + upper_counter = + ntohll(ctx->ctr_cb[0] & ctx->ctr_upper_mask); + upper_counter = htonll(upper_counter + 1); + upper_counter &= ctx->ctr_upper_mask; + ctx->ctr_cb[0] = + (ctx->ctr_cb[0] & ~(ctx->ctr_upper_mask)) | + upper_counter; + } + + /* + * XOR encrypted counter block with the current clear block. + */ + xor_block(blockp, lastp); + + if (out == NULL) { + if (ctx->ctr_remainder_len > 0) { + bcopy(lastp, ctx->ctr_copy_to, + ctx->ctr_remainder_len); + bcopy(lastp + ctx->ctr_remainder_len, datap, + need); + } + } else { + crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, + &out_data_1_len, &out_data_2, block_size); + + /* copy block to where it belongs */ + bcopy(lastp, out_data_1, out_data_1_len); + if (out_data_2 != NULL) { + bcopy(lastp + out_data_1_len, out_data_2, + block_size - out_data_1_len); + } + /* update offset */ + out->cd_offset += block_size; + } + + /* Update pointer to next block of data to be processed. */ + if (ctx->ctr_remainder_len != 0) { + datap += need; + ctx->ctr_remainder_len = 0; + } else { + datap += block_size; + } + + remainder = (size_t)&data[length] - (size_t)datap; + + /* Incomplete last block. */ + if (remainder > 0 && remainder < block_size) { + bcopy(datap, ctx->ctr_remainder, remainder); + ctx->ctr_remainder_len = remainder; + ctx->ctr_copy_to = datap; + goto out; + } + ctx->ctr_copy_to = NULL; + + } while (remainder > 0); + +out: + return (CRYPTO_SUCCESS); +} + +int +ctr_mode_final(ctr_ctx_t *ctx, crypto_data_t *out, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)) +{ + uint8_t *lastp; + void *iov_or_mp; + offset_t offset; + uint8_t *out_data_1; + uint8_t *out_data_2; + size_t out_data_1_len; + uint8_t *p; + int i; + + if (out->cd_length < ctx->ctr_remainder_len) + return (CRYPTO_DATA_LEN_RANGE); + + encrypt_block(ctx->ctr_keysched, (uint8_t *)ctx->ctr_cb, + (uint8_t *)ctx->ctr_tmp); + + lastp = (uint8_t *)ctx->ctr_tmp; + p = (uint8_t *)ctx->ctr_remainder; + for (i = 0; i < ctx->ctr_remainder_len; i++) { + p[i] ^= lastp[i]; + } + + crypto_init_ptrs(out, &iov_or_mp, &offset); + crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, + &out_data_1_len, &out_data_2, ctx->ctr_remainder_len); + + bcopy(p, out_data_1, out_data_1_len); + if (out_data_2 != NULL) { + bcopy((uint8_t *)p + out_data_1_len, + out_data_2, ctx->ctr_remainder_len - out_data_1_len); + } + out->cd_offset += ctx->ctr_remainder_len; + ctx->ctr_remainder_len = 0; + return (CRYPTO_SUCCESS); +} + +int +ctr_init_ctx(ctr_ctx_t *ctr_ctx, ulong_t count, uint8_t *cb, +void (*copy_block)(uint8_t *, uint8_t *)) +{ + uint64_t upper_mask = 0; + uint64_t lower_mask = 0; + + if (count == 0 || count > 128) { + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + /* upper 64 bits of the mask */ + if (count >= 64) { + count -= 64; + upper_mask = (count == 64) ? UINT64_MAX : (1ULL << count) - 1; + lower_mask = UINT64_MAX; + } else { + /* now the lower 63 bits */ + lower_mask = (1ULL << count) - 1; + } + ctr_ctx->ctr_lower_mask = htonll(lower_mask); + ctr_ctx->ctr_upper_mask = htonll(upper_mask); + + copy_block(cb, (uchar_t *)ctr_ctx->ctr_cb); + ctr_ctx->ctr_lastp = (uint8_t *)&ctr_ctx->ctr_cb[0]; + ctr_ctx->ctr_flags |= CTR_MODE; + return (CRYPTO_SUCCESS); +} + +/* ARGSUSED */ +void * +ctr_alloc_ctx(int kmflag) +{ + ctr_ctx_t *ctr_ctx; + + if ((ctr_ctx = kmem_zalloc(sizeof (ctr_ctx_t), kmflag)) == NULL) + return (NULL); + + ctr_ctx->ctr_flags = CTR_MODE; + return (ctr_ctx); +} diff --git a/module/icp/algs/modes/ecb.c b/module/icp/algs/modes/ecb.c new file mode 100644 index 000000000000..04e6c5eaa650 --- /dev/null +++ b/module/icp/algs/modes/ecb.c @@ -0,0 +1,143 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include + +/* + * Algorithm independent ECB functions. + */ +int +ecb_cipher_contiguous_blocks(ecb_ctx_t *ctx, char *data, size_t length, + crypto_data_t *out, size_t block_size, + int (*cipher)(const void *ks, const uint8_t *pt, uint8_t *ct)) +{ + size_t remainder = length; + size_t need = 0; + uint8_t *datap = (uint8_t *)data; + uint8_t *blockp; + uint8_t *lastp; + void *iov_or_mp; + offset_t offset; + uint8_t *out_data_1; + uint8_t *out_data_2; + size_t out_data_1_len; + + if (length + ctx->ecb_remainder_len < block_size) { + /* accumulate bytes here and return */ + bcopy(datap, + (uint8_t *)ctx->ecb_remainder + ctx->ecb_remainder_len, + length); + ctx->ecb_remainder_len += length; + ctx->ecb_copy_to = datap; + return (CRYPTO_SUCCESS); + } + + lastp = (uint8_t *)ctx->ecb_iv; + if (out != NULL) + crypto_init_ptrs(out, &iov_or_mp, &offset); + + do { + /* Unprocessed data from last call. */ + if (ctx->ecb_remainder_len > 0) { + need = block_size - ctx->ecb_remainder_len; + + if (need > remainder) + return (CRYPTO_DATA_LEN_RANGE); + + bcopy(datap, &((uint8_t *)ctx->ecb_remainder) + [ctx->ecb_remainder_len], need); + + blockp = (uint8_t *)ctx->ecb_remainder; + } else { + blockp = datap; + } + + if (out == NULL) { + cipher(ctx->ecb_keysched, blockp, blockp); + + ctx->ecb_lastp = blockp; + lastp = blockp; + + if (ctx->ecb_remainder_len > 0) { + bcopy(blockp, ctx->ecb_copy_to, + ctx->ecb_remainder_len); + bcopy(blockp + ctx->ecb_remainder_len, datap, + need); + } + } else { + cipher(ctx->ecb_keysched, blockp, lastp); + crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, + &out_data_1_len, &out_data_2, block_size); + + /* copy block to where it belongs */ + bcopy(lastp, out_data_1, out_data_1_len); + if (out_data_2 != NULL) { + bcopy(lastp + out_data_1_len, out_data_2, + block_size - out_data_1_len); + } + /* update offset */ + out->cd_offset += block_size; + } + + /* Update pointer to next block of data to be processed. */ + if (ctx->ecb_remainder_len != 0) { + datap += need; + ctx->ecb_remainder_len = 0; + } else { + datap += block_size; + } + + remainder = (size_t)&data[length] - (size_t)datap; + + /* Incomplete last block. */ + if (remainder > 0 && remainder < block_size) { + bcopy(datap, ctx->ecb_remainder, remainder); + ctx->ecb_remainder_len = remainder; + ctx->ecb_copy_to = datap; + goto out; + } + ctx->ecb_copy_to = NULL; + + } while (remainder > 0); + +out: + return (CRYPTO_SUCCESS); +} + +/* ARGSUSED */ +void * +ecb_alloc_ctx(int kmflag) +{ + ecb_ctx_t *ecb_ctx; + + if ((ecb_ctx = kmem_zalloc(sizeof (ecb_ctx_t), kmflag)) == NULL) + return (NULL); + + ecb_ctx->ecb_flags = ECB_MODE; + return (ecb_ctx); +} diff --git a/module/icp/algs/modes/gcm.c b/module/icp/algs/modes/gcm.c new file mode 100644 index 000000000000..9cd8ab1e9708 --- /dev/null +++ b/module/icp/algs/modes/gcm.c @@ -0,0 +1,748 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#ifdef __amd64 + +#ifdef _KERNEL +/* Workaround for no XMM kernel thread save/restore */ +#define KPREEMPT_DISABLE kpreempt_disable() +#define KPREEMPT_ENABLE kpreempt_enable() + +#else +#define KPREEMPT_DISABLE +#define KPREEMPT_ENABLE +#endif /* _KERNEL */ + +extern void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res); +static int intel_pclmulqdq_instruction_present(void); +#endif /* __amd64 */ + +struct aes_block { + uint64_t a; + uint64_t b; +}; + + +/* + * gcm_mul() + * Perform a carry-less multiplication (that is, use XOR instead of the + * multiply operator) on *x_in and *y and place the result in *res. + * + * Byte swap the input (*x_in and *y) and the output (*res). + * + * Note: x_in, y, and res all point to 16-byte numbers (an array of two + * 64-bit integers). + */ +void +gcm_mul(uint64_t *x_in, uint64_t *y, uint64_t *res) +{ +#ifdef __amd64 + if (intel_pclmulqdq_instruction_present()) { + KPREEMPT_DISABLE; + gcm_mul_pclmulqdq(x_in, y, res); + KPREEMPT_ENABLE; + } else +#endif /* __amd64 */ + { + static const uint64_t R = 0xe100000000000000ULL; + struct aes_block z = {0, 0}; + struct aes_block v; + uint64_t x; + int i, j; + + v.a = ntohll(y[0]); + v.b = ntohll(y[1]); + + for (j = 0; j < 2; j++) { + x = ntohll(x_in[j]); + for (i = 0; i < 64; i++, x <<= 1) { + if (x & 0x8000000000000000ULL) { + z.a ^= v.a; + z.b ^= v.b; + } + if (v.b & 1ULL) { + v.b = (v.a << 63)|(v.b >> 1); + v.a = (v.a >> 1) ^ R; + } else { + v.b = (v.a << 63)|(v.b >> 1); + v.a = v.a >> 1; + } + } + } + res[0] = htonll(z.a); + res[1] = htonll(z.b); + } +} + + +#define GHASH(c, d, t) \ + xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \ + gcm_mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \ + (uint64_t *)(void *)(t)); + + +/* + * Encrypt multiple blocks of data in GCM mode. Decrypt for GCM mode + * is done in another function. + */ +int +gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length, + crypto_data_t *out, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + size_t remainder = length; + size_t need = 0; + uint8_t *datap = (uint8_t *)data; + uint8_t *blockp; + uint8_t *lastp; + void *iov_or_mp; + offset_t offset; + uint8_t *out_data_1; + uint8_t *out_data_2; + size_t out_data_1_len; + uint64_t counter; + uint64_t counter_mask = ntohll(0x00000000ffffffffULL); + + if (length + ctx->gcm_remainder_len < block_size) { + /* accumulate bytes here and return */ + bcopy(datap, + (uint8_t *)ctx->gcm_remainder + ctx->gcm_remainder_len, + length); + ctx->gcm_remainder_len += length; + ctx->gcm_copy_to = datap; + return (CRYPTO_SUCCESS); + } + + lastp = (uint8_t *)ctx->gcm_cb; + if (out != NULL) + crypto_init_ptrs(out, &iov_or_mp, &offset); + + do { + /* Unprocessed data from last call. */ + if (ctx->gcm_remainder_len > 0) { + need = block_size - ctx->gcm_remainder_len; + + if (need > remainder) + return (CRYPTO_DATA_LEN_RANGE); + + bcopy(datap, &((uint8_t *)ctx->gcm_remainder) + [ctx->gcm_remainder_len], need); + + blockp = (uint8_t *)ctx->gcm_remainder; + } else { + blockp = datap; + } + + /* + * Increment counter. Counter bits are confined + * to the bottom 32 bits of the counter block. + */ + counter = ntohll(ctx->gcm_cb[1] & counter_mask); + counter = htonll(counter + 1); + counter &= counter_mask; + ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; + + encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, + (uint8_t *)ctx->gcm_tmp); + xor_block(blockp, (uint8_t *)ctx->gcm_tmp); + + lastp = (uint8_t *)ctx->gcm_tmp; + + ctx->gcm_processed_data_len += block_size; + + if (out == NULL) { + if (ctx->gcm_remainder_len > 0) { + bcopy(blockp, ctx->gcm_copy_to, + ctx->gcm_remainder_len); + bcopy(blockp + ctx->gcm_remainder_len, datap, + need); + } + } else { + crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, + &out_data_1_len, &out_data_2, block_size); + + /* copy block to where it belongs */ + if (out_data_1_len == block_size) { + copy_block(lastp, out_data_1); + } else { + bcopy(lastp, out_data_1, out_data_1_len); + if (out_data_2 != NULL) { + bcopy(lastp + out_data_1_len, + out_data_2, + block_size - out_data_1_len); + } + } + /* update offset */ + out->cd_offset += block_size; + } + + /* add ciphertext to the hash */ + GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash); + + /* Update pointer to next block of data to be processed. */ + if (ctx->gcm_remainder_len != 0) { + datap += need; + ctx->gcm_remainder_len = 0; + } else { + datap += block_size; + } + + remainder = (size_t)&data[length] - (size_t)datap; + + /* Incomplete last block. */ + if (remainder > 0 && remainder < block_size) { + bcopy(datap, ctx->gcm_remainder, remainder); + ctx->gcm_remainder_len = remainder; + ctx->gcm_copy_to = datap; + goto out; + } + ctx->gcm_copy_to = NULL; + + } while (remainder > 0); +out: + return (CRYPTO_SUCCESS); +} + +/* ARGSUSED */ +int +gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + uint64_t counter_mask = ntohll(0x00000000ffffffffULL); + uint8_t *ghash, *macp = NULL; + int i, rv; + + if (out->cd_length < + (ctx->gcm_remainder_len + ctx->gcm_tag_len)) { + return (CRYPTO_DATA_LEN_RANGE); + } + + ghash = (uint8_t *)ctx->gcm_ghash; + + if (ctx->gcm_remainder_len > 0) { + uint64_t counter; + uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp; + + /* + * Here is where we deal with data that is not a + * multiple of the block size. + */ + + /* + * Increment counter. + */ + counter = ntohll(ctx->gcm_cb[1] & counter_mask); + counter = htonll(counter + 1); + counter &= counter_mask; + ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; + + encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, + (uint8_t *)ctx->gcm_tmp); + + macp = (uint8_t *)ctx->gcm_remainder; + bzero(macp + ctx->gcm_remainder_len, + block_size - ctx->gcm_remainder_len); + + /* XOR with counter block */ + for (i = 0; i < ctx->gcm_remainder_len; i++) { + macp[i] ^= tmpp[i]; + } + + /* add ciphertext to the hash */ + GHASH(ctx, macp, ghash); + + ctx->gcm_processed_data_len += ctx->gcm_remainder_len; + } + + ctx->gcm_len_a_len_c[1] = + htonll(CRYPTO_BYTES2BITS(ctx->gcm_processed_data_len)); + GHASH(ctx, ctx->gcm_len_a_len_c, ghash); + encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0, + (uint8_t *)ctx->gcm_J0); + xor_block((uint8_t *)ctx->gcm_J0, ghash); + + if (ctx->gcm_remainder_len > 0) { + rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len); + if (rv != CRYPTO_SUCCESS) + return (rv); + } + out->cd_offset += ctx->gcm_remainder_len; + ctx->gcm_remainder_len = 0; + rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len); + if (rv != CRYPTO_SUCCESS) + return (rv); + out->cd_offset += ctx->gcm_tag_len; + + return (CRYPTO_SUCCESS); +} + +/* + * This will only deal with decrypting the last block of the input that + * might not be a multiple of block length. + */ +static void +gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + uint8_t *datap, *outp, *counterp; + uint64_t counter; + uint64_t counter_mask = ntohll(0x00000000ffffffffULL); + int i; + + /* + * Increment counter. + * Counter bits are confined to the bottom 32 bits + */ + counter = ntohll(ctx->gcm_cb[1] & counter_mask); + counter = htonll(counter + 1); + counter &= counter_mask; + ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; + + datap = (uint8_t *)ctx->gcm_remainder; + outp = &((ctx->gcm_pt_buf)[index]); + counterp = (uint8_t *)ctx->gcm_tmp; + + /* authentication tag */ + bzero((uint8_t *)ctx->gcm_tmp, block_size); + bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len); + + /* add ciphertext to the hash */ + GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash); + + /* decrypt remaining ciphertext */ + encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp); + + /* XOR with counter block */ + for (i = 0; i < ctx->gcm_remainder_len; i++) { + outp[i] = datap[i] ^ counterp[i]; + } +} + +/* ARGSUSED */ +int +gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length, + crypto_data_t *out, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + size_t new_len; + uint8_t *new; + + /* + * Copy contiguous ciphertext input blocks to plaintext buffer. + * Ciphertext will be decrypted in the final. + */ + if (length > 0) { + new_len = ctx->gcm_pt_buf_len + length; + new = vmem_alloc(new_len, ctx->gcm_kmflag); + bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len); + vmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len); + if (new == NULL) + return (CRYPTO_HOST_MEMORY); + + ctx->gcm_pt_buf = new; + ctx->gcm_pt_buf_len = new_len; + bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len], + length); + ctx->gcm_processed_data_len += length; + } + + ctx->gcm_remainder_len = 0; + return (CRYPTO_SUCCESS); +} + +int +gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + size_t pt_len; + size_t remainder; + uint8_t *ghash; + uint8_t *blockp; + uint8_t *cbp; + uint64_t counter; + uint64_t counter_mask = ntohll(0x00000000ffffffffULL); + int processed = 0, rv; + + ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len); + + pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len; + ghash = (uint8_t *)ctx->gcm_ghash; + blockp = ctx->gcm_pt_buf; + remainder = pt_len; + while (remainder > 0) { + /* Incomplete last block */ + if (remainder < block_size) { + bcopy(blockp, ctx->gcm_remainder, remainder); + ctx->gcm_remainder_len = remainder; + /* + * not expecting anymore ciphertext, just + * compute plaintext for the remaining input + */ + gcm_decrypt_incomplete_block(ctx, block_size, + processed, encrypt_block, xor_block); + ctx->gcm_remainder_len = 0; + goto out; + } + /* add ciphertext to the hash */ + GHASH(ctx, blockp, ghash); + + /* + * Increment counter. + * Counter bits are confined to the bottom 32 bits + */ + counter = ntohll(ctx->gcm_cb[1] & counter_mask); + counter = htonll(counter + 1); + counter &= counter_mask; + ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; + + cbp = (uint8_t *)ctx->gcm_tmp; + encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp); + + /* XOR with ciphertext */ + xor_block(cbp, blockp); + + processed += block_size; + blockp += block_size; + remainder -= block_size; + } +out: + ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(pt_len)); + GHASH(ctx, ctx->gcm_len_a_len_c, ghash); + encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0, + (uint8_t *)ctx->gcm_J0); + xor_block((uint8_t *)ctx->gcm_J0, ghash); + + /* compare the input authentication tag with what we calculated */ + if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) { + /* They don't match */ + return (CRYPTO_INVALID_MAC); + } else { + rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len); + if (rv != CRYPTO_SUCCESS) + return (rv); + out->cd_offset += pt_len; + } + return (CRYPTO_SUCCESS); +} + +static int +gcm_validate_args(CK_AES_GCM_PARAMS *gcm_param) +{ + size_t tag_len; + + /* + * Check the length of the authentication tag (in bits). + */ + tag_len = gcm_param->ulTagBits; + switch (tag_len) { + case 32: + case 64: + case 96: + case 104: + case 112: + case 120: + case 128: + break; + default: + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + + if (gcm_param->ulIvLen == 0) + return (CRYPTO_MECHANISM_PARAM_INVALID); + + return (CRYPTO_SUCCESS); +} + +static void +gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len, + gcm_ctx_t *ctx, size_t block_size, + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + uint8_t *cb; + ulong_t remainder = iv_len; + ulong_t processed = 0; + uint8_t *datap, *ghash; + uint64_t len_a_len_c[2]; + + ghash = (uint8_t *)ctx->gcm_ghash; + cb = (uint8_t *)ctx->gcm_cb; + if (iv_len == 12) { + bcopy(iv, cb, 12); + cb[12] = 0; + cb[13] = 0; + cb[14] = 0; + cb[15] = 1; + /* J0 will be used again in the final */ + copy_block(cb, (uint8_t *)ctx->gcm_J0); + } else { + /* GHASH the IV */ + do { + if (remainder < block_size) { + bzero(cb, block_size); + bcopy(&(iv[processed]), cb, remainder); + datap = (uint8_t *)cb; + remainder = 0; + } else { + datap = (uint8_t *)(&(iv[processed])); + processed += block_size; + remainder -= block_size; + } + GHASH(ctx, datap, ghash); + } while (remainder > 0); + + len_a_len_c[0] = 0; + len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(iv_len)); + GHASH(ctx, len_a_len_c, ctx->gcm_J0); + + /* J0 will be used again in the final */ + copy_block((uint8_t *)ctx->gcm_J0, (uint8_t *)cb); + } +} + +/* + * The following function is called at encrypt or decrypt init time + * for AES GCM mode. + */ +int +gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len, + unsigned char *auth_data, size_t auth_data_len, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + uint8_t *ghash, *datap, *authp; + size_t remainder, processed; + + /* encrypt zero block to get subkey H */ + bzero(ctx->gcm_H, sizeof (ctx->gcm_H)); + encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H, + (uint8_t *)ctx->gcm_H); + + gcm_format_initial_blocks(iv, iv_len, ctx, block_size, + copy_block, xor_block); + + authp = (uint8_t *)ctx->gcm_tmp; + ghash = (uint8_t *)ctx->gcm_ghash; + bzero(authp, block_size); + bzero(ghash, block_size); + + processed = 0; + remainder = auth_data_len; + do { + if (remainder < block_size) { + /* + * There's not a block full of data, pad rest of + * buffer with zero + */ + bzero(authp, block_size); + bcopy(&(auth_data[processed]), authp, remainder); + datap = (uint8_t *)authp; + remainder = 0; + } else { + datap = (uint8_t *)(&(auth_data[processed])); + processed += block_size; + remainder -= block_size; + } + + /* add auth data to the hash */ + GHASH(ctx, datap, ghash); + + } while (remainder > 0); + + return (CRYPTO_SUCCESS); +} + +int +gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + int rv; + CK_AES_GCM_PARAMS *gcm_param; + + if (param != NULL) { + gcm_param = (CK_AES_GCM_PARAMS *)(void *)param; + + if ((rv = gcm_validate_args(gcm_param)) != 0) { + return (rv); + } + + gcm_ctx->gcm_tag_len = gcm_param->ulTagBits; + gcm_ctx->gcm_tag_len >>= 3; + gcm_ctx->gcm_processed_data_len = 0; + + /* these values are in bits */ + gcm_ctx->gcm_len_a_len_c[0] + = htonll(CRYPTO_BYTES2BITS(gcm_param->ulAADLen)); + + rv = CRYPTO_SUCCESS; + gcm_ctx->gcm_flags |= GCM_MODE; + } else { + rv = CRYPTO_MECHANISM_PARAM_INVALID; + goto out; + } + + if (gcm_init(gcm_ctx, gcm_param->pIv, gcm_param->ulIvLen, + gcm_param->pAAD, gcm_param->ulAADLen, block_size, + encrypt_block, copy_block, xor_block) != 0) { + rv = CRYPTO_MECHANISM_PARAM_INVALID; + } +out: + return (rv); +} + +int +gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)) +{ + int rv; + CK_AES_GMAC_PARAMS *gmac_param; + + if (param != NULL) { + gmac_param = (CK_AES_GMAC_PARAMS *)(void *)param; + + gcm_ctx->gcm_tag_len = CRYPTO_BITS2BYTES(AES_GMAC_TAG_BITS); + gcm_ctx->gcm_processed_data_len = 0; + + /* these values are in bits */ + gcm_ctx->gcm_len_a_len_c[0] + = htonll(CRYPTO_BYTES2BITS(gmac_param->ulAADLen)); + + rv = CRYPTO_SUCCESS; + gcm_ctx->gcm_flags |= GMAC_MODE; + } else { + rv = CRYPTO_MECHANISM_PARAM_INVALID; + goto out; + } + + if (gcm_init(gcm_ctx, gmac_param->pIv, AES_GMAC_IV_LEN, + gmac_param->pAAD, gmac_param->ulAADLen, block_size, + encrypt_block, copy_block, xor_block) != 0) { + rv = CRYPTO_MECHANISM_PARAM_INVALID; + } +out: + return (rv); +} + +void * +gcm_alloc_ctx(int kmflag) +{ + gcm_ctx_t *gcm_ctx; + + if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL) + return (NULL); + + gcm_ctx->gcm_flags = GCM_MODE; + return (gcm_ctx); +} + +void * +gmac_alloc_ctx(int kmflag) +{ + gcm_ctx_t *gcm_ctx; + + if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL) + return (NULL); + + gcm_ctx->gcm_flags = GMAC_MODE; + return (gcm_ctx); +} + +void +gcm_set_kmflag(gcm_ctx_t *ctx, int kmflag) +{ + ctx->gcm_kmflag = kmflag; +} + + +#ifdef __amd64 + +#define INTEL_PCLMULQDQ_FLAG (1 << 1) + +/* + * Return 1 if executing on Intel with PCLMULQDQ instructions, + * otherwise 0 (i.e., Intel without PCLMULQDQ or AMD64). + * Cache the result, as the CPU can't change. + * + * Note: the userland version uses getisax(). The kernel version uses + * is_x86_featureset(). + */ +static int +intel_pclmulqdq_instruction_present(void) +{ + static int cached_result = -1; + unsigned eax, ebx, ecx, edx; + unsigned func, subfunc; + + if (cached_result == -1) { /* first time */ + /* check for an intel cpu */ + func = 0; + subfunc = 0; + + __asm__ __volatile__( + "cpuid" + : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) + : "a"(func), "c"(subfunc)); + + if (memcmp((char *) (&ebx), "Genu", 4) == 0 && + memcmp((char *) (&edx), "ineI", 4) == 0 && + memcmp((char *) (&ecx), "ntel", 4) == 0) { + + func = 1; + subfunc = 0; + + /* check for aes-ni instruction set */ + __asm__ __volatile__( + "cpuid" + : "=a" (eax), "=b" (ebx), "=c" (ecx), "=d" (edx) + : "a"(func), "c"(subfunc)); + + cached_result = !!(ecx & INTEL_PCLMULQDQ_FLAG); + } else { + cached_result = 0; + } + } + + return (cached_result); +} + +#endif /* __amd64 */ diff --git a/module/icp/algs/modes/modes.c b/module/icp/algs/modes/modes.c new file mode 100644 index 000000000000..1d33c4268816 --- /dev/null +++ b/module/icp/algs/modes/modes.c @@ -0,0 +1,159 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include + +/* + * Initialize by setting iov_or_mp to point to the current iovec or mp, + * and by setting current_offset to an offset within the current iovec or mp. + */ +void +crypto_init_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset) +{ + offset_t offset; + + switch (out->cd_format) { + case CRYPTO_DATA_RAW: + *current_offset = out->cd_offset; + break; + + case CRYPTO_DATA_UIO: { + uio_t *uiop = out->cd_uio; + uintptr_t vec_idx; + + offset = out->cd_offset; + for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && + offset >= uiop->uio_iov[vec_idx].iov_len; + offset -= uiop->uio_iov[vec_idx++].iov_len) + ; + + *current_offset = offset; + *iov_or_mp = (void *)vec_idx; + break; + } + } /* end switch */ +} + +/* + * Get pointers for where in the output to copy a block of encrypted or + * decrypted data. The iov_or_mp argument stores a pointer to the current + * iovec or mp, and offset stores an offset into the current iovec or mp. + */ +void +crypto_get_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset, + uint8_t **out_data_1, size_t *out_data_1_len, uint8_t **out_data_2, + size_t amt) +{ + offset_t offset; + + switch (out->cd_format) { + case CRYPTO_DATA_RAW: { + iovec_t *iov; + + offset = *current_offset; + iov = &out->cd_raw; + if ((offset + amt) <= iov->iov_len) { + /* one block fits */ + *out_data_1 = (uint8_t *)iov->iov_base + offset; + *out_data_1_len = amt; + *out_data_2 = NULL; + *current_offset = offset + amt; + } + break; + } + + case CRYPTO_DATA_UIO: { + uio_t *uio = out->cd_uio; + iovec_t *iov; + offset_t offset; + uintptr_t vec_idx; + uint8_t *p; + + offset = *current_offset; + vec_idx = (uintptr_t)(*iov_or_mp); + iov = (iovec_t *)&uio->uio_iov[vec_idx]; + p = (uint8_t *)iov->iov_base + offset; + *out_data_1 = p; + + if (offset + amt <= iov->iov_len) { + /* can fit one block into this iov */ + *out_data_1_len = amt; + *out_data_2 = NULL; + *current_offset = offset + amt; + } else { + /* one block spans two iovecs */ + *out_data_1_len = iov->iov_len - offset; + if (vec_idx == uio->uio_iovcnt) + return; + vec_idx++; + iov = (iovec_t *)&uio->uio_iov[vec_idx]; + *out_data_2 = (uint8_t *)iov->iov_base; + *current_offset = amt - *out_data_1_len; + } + *iov_or_mp = (void *)vec_idx; + break; + } + } /* end switch */ +} + +void +crypto_free_mode_ctx(void *ctx) +{ + common_ctx_t *common_ctx = (common_ctx_t *)ctx; + + switch (common_ctx->cc_flags & + (ECB_MODE|CBC_MODE|CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) { + case ECB_MODE: + kmem_free(common_ctx, sizeof (ecb_ctx_t)); + break; + + case CBC_MODE: + kmem_free(common_ctx, sizeof (cbc_ctx_t)); + break; + + case CTR_MODE: + kmem_free(common_ctx, sizeof (ctr_ctx_t)); + break; + + case CCM_MODE: + if (((ccm_ctx_t *)ctx)->ccm_pt_buf != NULL) + vmem_free(((ccm_ctx_t *)ctx)->ccm_pt_buf, + ((ccm_ctx_t *)ctx)->ccm_data_len); + + kmem_free(ctx, sizeof (ccm_ctx_t)); + break; + + case GCM_MODE: + case GMAC_MODE: + if (((gcm_ctx_t *)ctx)->gcm_pt_buf != NULL) + vmem_free(((gcm_ctx_t *)ctx)->gcm_pt_buf, + ((gcm_ctx_t *)ctx)->gcm_pt_buf_len); + + kmem_free(ctx, sizeof (gcm_ctx_t)); + } +} diff --git a/module/icp/algs/sha1/sha1.c b/module/icp/algs/sha1/sha1.c new file mode 100644 index 000000000000..b826c54ade7b --- /dev/null +++ b/module/icp/algs/sha1/sha1.c @@ -0,0 +1,663 @@ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * The basic framework for this code came from the reference + * implementation for MD5. That implementation is Copyright (C) + * 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved. + * + * License to copy and use this software is granted provided that it + * is identified as the "RSA Data Security, Inc. MD5 Message-Digest + * Algorithm" in all material mentioning or referencing this software + * or this function. + * + * License is also granted to make and use derivative works provided + * that such works are identified as "derived from the RSA Data + * Security, Inc. MD5 Message-Digest Algorithm" in all material + * mentioning or referencing the derived work. + * + * RSA Data Security, Inc. makes no representations concerning either + * the merchantability of this software or the suitability of this + * software for any particular purpose. It is provided "as is" + * without express or implied warranty of any kind. + * + * These notices must be retained in any copies of any part of this + * documentation and/or software. + * + * NOTE: Cleaned-up and optimized, version of SHA1, based on the FIPS 180-1 + * standard, available at http://www.itl.nist.gov/fipspubs/fip180-1.htm + * Not as fast as one would like -- further optimizations are encouraged + * and appreciated. + */ + +#include +#include +#include + +#ifdef _LITTLE_ENDIAN +#include +#define HAVE_HTONL +#endif + +#define _RESTRICT_KYWD + +static void Encode(uint8_t *, const uint32_t *, size_t); + +#if defined(__amd64) + +#define SHA1_TRANSFORM(ctx, in) sha1_block_data_order((ctx), (in), 1) +#define SHA1_TRANSFORM_BLOCKS(ctx, in, num) sha1_block_data_order((ctx), \ + (in), (num)) + +void sha1_block_data_order(SHA1_CTX *ctx, const void *inpp, size_t num_blocks); + +#else + +#define SHA1_TRANSFORM(ctx, in) SHA1Transform((ctx), (in)) + +static void SHA1Transform(SHA1_CTX *, const uint8_t *); + +#endif + + +static uint8_t PADDING[64] = { 0x80, /* all zeros */ }; + +/* + * F, G, and H are the basic SHA1 functions. + */ +#define F(b, c, d) (((b) & (c)) | ((~b) & (d))) +#define G(b, c, d) ((b) ^ (c) ^ (d)) +#define H(b, c, d) (((b) & (c)) | (((b)|(c)) & (d))) + +/* + * ROTATE_LEFT rotates x left n bits. + */ + +#if defined(__GNUC__) && defined(_LP64) +static __inline__ uint64_t +ROTATE_LEFT(uint64_t value, uint32_t n) +{ + uint32_t t32; + + t32 = (uint32_t)value; + return ((t32 << n) | (t32 >> (32 - n))); +} + +#else + +#define ROTATE_LEFT(x, n) \ + (((x) << (n)) | ((x) >> ((sizeof (x) * NBBY)-(n)))) + +#endif + + +/* + * SHA1Init() + * + * purpose: initializes the sha1 context and begins and sha1 digest operation + * input: SHA1_CTX * : the context to initializes. + * output: void + */ + +void +SHA1Init(SHA1_CTX *ctx) +{ + ctx->count[0] = ctx->count[1] = 0; + + /* + * load magic initialization constants. Tell lint + * that these constants are unsigned by using U. + */ + + ctx->state[0] = 0x67452301U; + ctx->state[1] = 0xefcdab89U; + ctx->state[2] = 0x98badcfeU; + ctx->state[3] = 0x10325476U; + ctx->state[4] = 0xc3d2e1f0U; +} + +void +SHA1Update(SHA1_CTX *ctx, const void *inptr, size_t input_len) +{ + uint32_t i, buf_index, buf_len; + const uint8_t *input = inptr; +#if defined(__amd64) + uint32_t block_count; +#endif /* __amd64 */ + + /* check for noop */ + if (input_len == 0) + return; + + /* compute number of bytes mod 64 */ + buf_index = (ctx->count[1] >> 3) & 0x3F; + + /* update number of bits */ + if ((ctx->count[1] += (input_len << 3)) < (input_len << 3)) + ctx->count[0]++; + + ctx->count[0] += (input_len >> 29); + + buf_len = 64 - buf_index; + + /* transform as many times as possible */ + i = 0; + if (input_len >= buf_len) { + + /* + * general optimization: + * + * only do initial bcopy() and SHA1Transform() if + * buf_index != 0. if buf_index == 0, we're just + * wasting our time doing the bcopy() since there + * wasn't any data left over from a previous call to + * SHA1Update(). + */ + + if (buf_index) { + bcopy(input, &ctx->buf_un.buf8[buf_index], buf_len); + SHA1_TRANSFORM(ctx, ctx->buf_un.buf8); + i = buf_len; + } + +#if !defined(__amd64) + for (; i + 63 < input_len; i += 64) + SHA1_TRANSFORM(ctx, &input[i]); +#else + block_count = (input_len - i) >> 6; + if (block_count > 0) { + SHA1_TRANSFORM_BLOCKS(ctx, &input[i], block_count); + i += block_count << 6; + } +#endif /* !__amd64 */ + + /* + * general optimization: + * + * if i and input_len are the same, return now instead + * of calling bcopy(), since the bcopy() in this case + * will be an expensive nop. + */ + + if (input_len == i) + return; + + buf_index = 0; + } + + /* buffer remaining input */ + bcopy(&input[i], &ctx->buf_un.buf8[buf_index], input_len - i); +} + +/* + * SHA1Final() + * + * purpose: ends an sha1 digest operation, finalizing the message digest and + * zeroing the context. + * input: uchar_t * : A buffer to store the digest. + * : The function actually uses void* because many + * : callers pass things other than uchar_t here. + * SHA1_CTX * : the context to finalize, save, and zero + * output: void + */ + +void +SHA1Final(void *digest, SHA1_CTX *ctx) +{ + uint8_t bitcount_be[sizeof (ctx->count)]; + uint32_t index = (ctx->count[1] >> 3) & 0x3f; + + /* store bit count, big endian */ + Encode(bitcount_be, ctx->count, sizeof (bitcount_be)); + + /* pad out to 56 mod 64 */ + SHA1Update(ctx, PADDING, ((index < 56) ? 56 : 120) - index); + + /* append length (before padding) */ + SHA1Update(ctx, bitcount_be, sizeof (bitcount_be)); + + /* store state in digest */ + Encode(digest, ctx->state, sizeof (ctx->state)); + + /* zeroize sensitive information */ + bzero(ctx, sizeof (*ctx)); +} + + +#if !defined(__amd64) + +typedef uint32_t sha1word; + +/* + * sparc optimization: + * + * on the sparc, we can load big endian 32-bit data easily. note that + * special care must be taken to ensure the address is 32-bit aligned. + * in the interest of speed, we don't check to make sure, since + * careful programming can guarantee this for us. + */ + +#if defined(_BIG_ENDIAN) +#define LOAD_BIG_32(addr) (*(uint32_t *)(addr)) + +#elif defined(HAVE_HTONL) +#define LOAD_BIG_32(addr) htonl(*((uint32_t *)(addr))) + +#else +/* little endian -- will work on big endian, but slowly */ +#define LOAD_BIG_32(addr) \ + (((addr)[0] << 24) | ((addr)[1] << 16) | ((addr)[2] << 8) | (addr)[3]) +#endif /* _BIG_ENDIAN */ + +/* + * SHA1Transform() + */ +#if defined(W_ARRAY) +#define W(n) w[n] +#else /* !defined(W_ARRAY) */ +#define W(n) w_ ## n +#endif /* !defined(W_ARRAY) */ + +void /* CSTYLED */ +SHA1Transform(SHA1_CTX *ctx, const uint8_t blk[64]) +{ + /* CSTYLED */ + sha1word a = ctx->state[0]; + sha1word b = ctx->state[1]; + sha1word c = ctx->state[2]; + sha1word d = ctx->state[3]; + sha1word e = ctx->state[4]; + +#if defined(W_ARRAY) + sha1word w[16]; +#else /* !defined(W_ARRAY) */ + sha1word w_0, w_1, w_2, w_3, w_4, w_5, w_6, w_7; + sha1word w_8, w_9, w_10, w_11, w_12, w_13, w_14, w_15; +#endif /* !defined(W_ARRAY) */ + + W(0) = LOAD_BIG_32((void *)(blk + 0)); + W(1) = LOAD_BIG_32((void *)(blk + 4)); + W(2) = LOAD_BIG_32((void *)(blk + 8)); + W(3) = LOAD_BIG_32((void *)(blk + 12)); + W(4) = LOAD_BIG_32((void *)(blk + 16)); + W(5) = LOAD_BIG_32((void *)(blk + 20)); + W(6) = LOAD_BIG_32((void *)(blk + 24)); + W(7) = LOAD_BIG_32((void *)(blk + 28)); + W(8) = LOAD_BIG_32((void *)(blk + 32)); + W(9) = LOAD_BIG_32((void *)(blk + 36)); + W(10) = LOAD_BIG_32((void *)(blk + 40)); + W(11) = LOAD_BIG_32((void *)(blk + 44)); + W(12) = LOAD_BIG_32((void *)(blk + 48)); + W(13) = LOAD_BIG_32((void *)(blk + 52)); + W(14) = LOAD_BIG_32((void *)(blk + 56)); + W(15) = LOAD_BIG_32((void *)(blk + 60)); + + /* + * general optimization: + * + * even though this approach is described in the standard as + * being slower algorithmically, it is 30-40% faster than the + * "faster" version under SPARC, because this version has more + * of the constraints specified at compile-time and uses fewer + * variables (and therefore has better register utilization) + * than its "speedier" brother. (i've tried both, trust me) + * + * for either method given in the spec, there is an "assignment" + * phase where the following takes place: + * + * tmp = (main_computation); + * e = d; d = c; c = rotate_left(b, 30); b = a; a = tmp; + * + * we can make the algorithm go faster by not doing this work, + * but just pretending that `d' is now `e', etc. this works + * really well and obviates the need for a temporary variable. + * however, we still explicitly perform the rotate action, + * since it is cheaper on SPARC to do it once than to have to + * do it over and over again. + */ + + /* round 1 */ + e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(0) + SHA1_CONST(0); /* 0 */ + b = ROTATE_LEFT(b, 30); + + d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(1) + SHA1_CONST(0); /* 1 */ + a = ROTATE_LEFT(a, 30); + + c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(2) + SHA1_CONST(0); /* 2 */ + e = ROTATE_LEFT(e, 30); + + b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(3) + SHA1_CONST(0); /* 3 */ + d = ROTATE_LEFT(d, 30); + + a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(4) + SHA1_CONST(0); /* 4 */ + c = ROTATE_LEFT(c, 30); + + e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(5) + SHA1_CONST(0); /* 5 */ + b = ROTATE_LEFT(b, 30); + + d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(6) + SHA1_CONST(0); /* 6 */ + a = ROTATE_LEFT(a, 30); + + c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(7) + SHA1_CONST(0); /* 7 */ + e = ROTATE_LEFT(e, 30); + + b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(8) + SHA1_CONST(0); /* 8 */ + d = ROTATE_LEFT(d, 30); + + a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(9) + SHA1_CONST(0); /* 9 */ + c = ROTATE_LEFT(c, 30); + + e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(10) + SHA1_CONST(0); /* 10 */ + b = ROTATE_LEFT(b, 30); + + d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(11) + SHA1_CONST(0); /* 11 */ + a = ROTATE_LEFT(a, 30); + + c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(12) + SHA1_CONST(0); /* 12 */ + e = ROTATE_LEFT(e, 30); + + b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(13) + SHA1_CONST(0); /* 13 */ + d = ROTATE_LEFT(d, 30); + + a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(14) + SHA1_CONST(0); /* 14 */ + c = ROTATE_LEFT(c, 30); + + e = ROTATE_LEFT(a, 5) + F(b, c, d) + e + W(15) + SHA1_CONST(0); /* 15 */ + b = ROTATE_LEFT(b, 30); + + W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1); /* 16 */ + d = ROTATE_LEFT(e, 5) + F(a, b, c) + d + W(0) + SHA1_CONST(0); + a = ROTATE_LEFT(a, 30); + + W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1); /* 17 */ + c = ROTATE_LEFT(d, 5) + F(e, a, b) + c + W(1) + SHA1_CONST(0); + e = ROTATE_LEFT(e, 30); + + W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1); /* 18 */ + b = ROTATE_LEFT(c, 5) + F(d, e, a) + b + W(2) + SHA1_CONST(0); + d = ROTATE_LEFT(d, 30); + + W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1); /* 19 */ + a = ROTATE_LEFT(b, 5) + F(c, d, e) + a + W(3) + SHA1_CONST(0); + c = ROTATE_LEFT(c, 30); + + /* round 2 */ + W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1); /* 20 */ + e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(4) + SHA1_CONST(1); + b = ROTATE_LEFT(b, 30); + + W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1); /* 21 */ + d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(5) + SHA1_CONST(1); + a = ROTATE_LEFT(a, 30); + + W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1); /* 22 */ + c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(6) + SHA1_CONST(1); + e = ROTATE_LEFT(e, 30); + + W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1); /* 23 */ + b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(7) + SHA1_CONST(1); + d = ROTATE_LEFT(d, 30); + + W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1); /* 24 */ + a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(8) + SHA1_CONST(1); + c = ROTATE_LEFT(c, 30); + + W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1); /* 25 */ + e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(9) + SHA1_CONST(1); + b = ROTATE_LEFT(b, 30); + + W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1); /* 26 */ + d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(10) + SHA1_CONST(1); + a = ROTATE_LEFT(a, 30); + + W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1); /* 27 */ + c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(11) + SHA1_CONST(1); + e = ROTATE_LEFT(e, 30); + + W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1); /* 28 */ + b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(12) + SHA1_CONST(1); + d = ROTATE_LEFT(d, 30); + + W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1); /* 29 */ + a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(13) + SHA1_CONST(1); + c = ROTATE_LEFT(c, 30); + + W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1); /* 30 */ + e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(14) + SHA1_CONST(1); + b = ROTATE_LEFT(b, 30); + + W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1); /* 31 */ + d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(15) + SHA1_CONST(1); + a = ROTATE_LEFT(a, 30); + + W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1); /* 32 */ + c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(0) + SHA1_CONST(1); + e = ROTATE_LEFT(e, 30); + + W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1); /* 33 */ + b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(1) + SHA1_CONST(1); + d = ROTATE_LEFT(d, 30); + + W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1); /* 34 */ + a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(2) + SHA1_CONST(1); + c = ROTATE_LEFT(c, 30); + + W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1); /* 35 */ + e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(3) + SHA1_CONST(1); + b = ROTATE_LEFT(b, 30); + + W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1); /* 36 */ + d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(4) + SHA1_CONST(1); + a = ROTATE_LEFT(a, 30); + + W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1); /* 37 */ + c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(5) + SHA1_CONST(1); + e = ROTATE_LEFT(e, 30); + + W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1); /* 38 */ + b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(6) + SHA1_CONST(1); + d = ROTATE_LEFT(d, 30); + + W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1); /* 39 */ + a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(7) + SHA1_CONST(1); + c = ROTATE_LEFT(c, 30); + + /* round 3 */ + W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1); /* 40 */ + e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(8) + SHA1_CONST(2); + b = ROTATE_LEFT(b, 30); + + W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1); /* 41 */ + d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(9) + SHA1_CONST(2); + a = ROTATE_LEFT(a, 30); + + W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1); /* 42 */ + c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(10) + SHA1_CONST(2); + e = ROTATE_LEFT(e, 30); + + W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1); /* 43 */ + b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(11) + SHA1_CONST(2); + d = ROTATE_LEFT(d, 30); + + W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1); /* 44 */ + a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(12) + SHA1_CONST(2); + c = ROTATE_LEFT(c, 30); + + W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1); /* 45 */ + e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(13) + SHA1_CONST(2); + b = ROTATE_LEFT(b, 30); + + W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1); /* 46 */ + d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(14) + SHA1_CONST(2); + a = ROTATE_LEFT(a, 30); + + W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1); /* 47 */ + c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(15) + SHA1_CONST(2); + e = ROTATE_LEFT(e, 30); + + W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1); /* 48 */ + b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(0) + SHA1_CONST(2); + d = ROTATE_LEFT(d, 30); + + W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1); /* 49 */ + a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(1) + SHA1_CONST(2); + c = ROTATE_LEFT(c, 30); + + W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1); /* 50 */ + e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(2) + SHA1_CONST(2); + b = ROTATE_LEFT(b, 30); + + W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1); /* 51 */ + d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(3) + SHA1_CONST(2); + a = ROTATE_LEFT(a, 30); + + W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1); /* 52 */ + c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(4) + SHA1_CONST(2); + e = ROTATE_LEFT(e, 30); + + W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1); /* 53 */ + b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(5) + SHA1_CONST(2); + d = ROTATE_LEFT(d, 30); + + W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1); /* 54 */ + a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(6) + SHA1_CONST(2); + c = ROTATE_LEFT(c, 30); + + W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1); /* 55 */ + e = ROTATE_LEFT(a, 5) + H(b, c, d) + e + W(7) + SHA1_CONST(2); + b = ROTATE_LEFT(b, 30); + + W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1); /* 56 */ + d = ROTATE_LEFT(e, 5) + H(a, b, c) + d + W(8) + SHA1_CONST(2); + a = ROTATE_LEFT(a, 30); + + W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1); /* 57 */ + c = ROTATE_LEFT(d, 5) + H(e, a, b) + c + W(9) + SHA1_CONST(2); + e = ROTATE_LEFT(e, 30); + + W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1); /* 58 */ + b = ROTATE_LEFT(c, 5) + H(d, e, a) + b + W(10) + SHA1_CONST(2); + d = ROTATE_LEFT(d, 30); + + W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1); /* 59 */ + a = ROTATE_LEFT(b, 5) + H(c, d, e) + a + W(11) + SHA1_CONST(2); + c = ROTATE_LEFT(c, 30); + + /* round 4 */ + W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1); /* 60 */ + e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(12) + SHA1_CONST(3); + b = ROTATE_LEFT(b, 30); + + W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1); /* 61 */ + d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(13) + SHA1_CONST(3); + a = ROTATE_LEFT(a, 30); + + W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1); /* 62 */ + c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(14) + SHA1_CONST(3); + e = ROTATE_LEFT(e, 30); + + W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1); /* 63 */ + b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(15) + SHA1_CONST(3); + d = ROTATE_LEFT(d, 30); + + W(0) = ROTATE_LEFT((W(13) ^ W(8) ^ W(2) ^ W(0)), 1); /* 64 */ + a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(0) + SHA1_CONST(3); + c = ROTATE_LEFT(c, 30); + + W(1) = ROTATE_LEFT((W(14) ^ W(9) ^ W(3) ^ W(1)), 1); /* 65 */ + e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(1) + SHA1_CONST(3); + b = ROTATE_LEFT(b, 30); + + W(2) = ROTATE_LEFT((W(15) ^ W(10) ^ W(4) ^ W(2)), 1); /* 66 */ + d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(2) + SHA1_CONST(3); + a = ROTATE_LEFT(a, 30); + + W(3) = ROTATE_LEFT((W(0) ^ W(11) ^ W(5) ^ W(3)), 1); /* 67 */ + c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(3) + SHA1_CONST(3); + e = ROTATE_LEFT(e, 30); + + W(4) = ROTATE_LEFT((W(1) ^ W(12) ^ W(6) ^ W(4)), 1); /* 68 */ + b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(4) + SHA1_CONST(3); + d = ROTATE_LEFT(d, 30); + + W(5) = ROTATE_LEFT((W(2) ^ W(13) ^ W(7) ^ W(5)), 1); /* 69 */ + a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(5) + SHA1_CONST(3); + c = ROTATE_LEFT(c, 30); + + W(6) = ROTATE_LEFT((W(3) ^ W(14) ^ W(8) ^ W(6)), 1); /* 70 */ + e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(6) + SHA1_CONST(3); + b = ROTATE_LEFT(b, 30); + + W(7) = ROTATE_LEFT((W(4) ^ W(15) ^ W(9) ^ W(7)), 1); /* 71 */ + d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(7) + SHA1_CONST(3); + a = ROTATE_LEFT(a, 30); + + W(8) = ROTATE_LEFT((W(5) ^ W(0) ^ W(10) ^ W(8)), 1); /* 72 */ + c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(8) + SHA1_CONST(3); + e = ROTATE_LEFT(e, 30); + + W(9) = ROTATE_LEFT((W(6) ^ W(1) ^ W(11) ^ W(9)), 1); /* 73 */ + b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(9) + SHA1_CONST(3); + d = ROTATE_LEFT(d, 30); + + W(10) = ROTATE_LEFT((W(7) ^ W(2) ^ W(12) ^ W(10)), 1); /* 74 */ + a = ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(10) + SHA1_CONST(3); + c = ROTATE_LEFT(c, 30); + + W(11) = ROTATE_LEFT((W(8) ^ W(3) ^ W(13) ^ W(11)), 1); /* 75 */ + e = ROTATE_LEFT(a, 5) + G(b, c, d) + e + W(11) + SHA1_CONST(3); + b = ROTATE_LEFT(b, 30); + + W(12) = ROTATE_LEFT((W(9) ^ W(4) ^ W(14) ^ W(12)), 1); /* 76 */ + d = ROTATE_LEFT(e, 5) + G(a, b, c) + d + W(12) + SHA1_CONST(3); + a = ROTATE_LEFT(a, 30); + + W(13) = ROTATE_LEFT((W(10) ^ W(5) ^ W(15) ^ W(13)), 1); /* 77 */ + c = ROTATE_LEFT(d, 5) + G(e, a, b) + c + W(13) + SHA1_CONST(3); + e = ROTATE_LEFT(e, 30); + + W(14) = ROTATE_LEFT((W(11) ^ W(6) ^ W(0) ^ W(14)), 1); /* 78 */ + b = ROTATE_LEFT(c, 5) + G(d, e, a) + b + W(14) + SHA1_CONST(3); + d = ROTATE_LEFT(d, 30); + + W(15) = ROTATE_LEFT((W(12) ^ W(7) ^ W(1) ^ W(15)), 1); /* 79 */ + + ctx->state[0] += ROTATE_LEFT(b, 5) + G(c, d, e) + a + W(15) + + SHA1_CONST(3); + ctx->state[1] += b; + ctx->state[2] += ROTATE_LEFT(c, 30); + ctx->state[3] += d; + ctx->state[4] += e; + + /* zeroize sensitive information */ + W(0) = W(1) = W(2) = W(3) = W(4) = W(5) = W(6) = W(7) = W(8) = 0; + W(9) = W(10) = W(11) = W(12) = W(13) = W(14) = W(15) = 0; +} +#endif /* !__amd64 */ + + +/* + * Encode() + * + * purpose: to convert a list of numbers from little endian to big endian + * input: uint8_t * : place to store the converted big endian numbers + * uint32_t * : place to get numbers to convert from + * size_t : the length of the input in bytes + * output: void + */ + +static void +Encode(uint8_t *_RESTRICT_KYWD output, const uint32_t *_RESTRICT_KYWD input, + size_t len) +{ + size_t i, j; + + for (i = 0, j = 0; j < len; i++, j += 4) { + output[j] = (input[i] >> 24) & 0xff; + output[j + 1] = (input[i] >> 16) & 0xff; + output[j + 2] = (input[i] >> 8) & 0xff; + output[j + 3] = input[i] & 0xff; + } +} diff --git a/module/icp/algs/sha2/sha2.c b/module/icp/algs/sha2/sha2.c new file mode 100644 index 000000000000..792ca8825cbe --- /dev/null +++ b/module/icp/algs/sha2/sha2.c @@ -0,0 +1,495 @@ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +/* + * Copyright 2013 Saso Kiselkov. All rights reserved. + */ + +/* + * The basic framework for this code came from the reference + * implementation for MD5. That implementation is Copyright (C) + * 1991-2, RSA Data Security, Inc. Created 1991. All rights reserved. + * + * License to copy and use this software is granted provided that it + * is identified as the "RSA Data Security, Inc. MD5 Message-Digest + * Algorithm" in all material mentioning or referencing this software + * or this function. + * + * License is also granted to make and use derivative works provided + * that such works are identified as "derived from the RSA Data + * Security, Inc. MD5 Message-Digest Algorithm" in all material + * mentioning or referencing the derived work. + * + * RSA Data Security, Inc. makes no representations concerning either + * the merchantability of this software or the suitability of this + * software for any particular purpose. It is provided "as is" + * without express or implied warranty of any kind. + * + * These notices must be retained in any copies of any part of this + * documentation and/or software. + * + * NOTE: Cleaned-up and optimized, version of SHA2, based on the FIPS 180-2 + * standard, available at + * http://csrc.nist.gov/publications/fips/fips180-2/fips180-2.pdf + * Not as fast as one would like -- further optimizations are encouraged + * and appreciated. + */ + +#include +#define _SHA2_IMPL +#include +#include + +#define _RESTRICT_KYWD + +#ifdef _LITTLE_ENDIAN +#include +#define HAVE_HTONL +#endif + +static void Encode(uint8_t *, uint32_t *, size_t); + +#if defined(__amd64) +#define SHA256Transform(ctx, in) SHA256TransformBlocks((ctx), (in), 1) +void SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num); +#else +static void SHA256Transform(SHA2_CTX *, const uint8_t *); +#endif /* __amd64 */ + +static uint8_t PADDING[128] = { 0x80, /* all zeros */ }; + +/* Ch and Maj are the basic SHA2 functions. */ +#define Ch(b, c, d) (((b) & (c)) ^ ((~b) & (d))) +#define Maj(b, c, d) (((b) & (c)) ^ ((b) & (d)) ^ ((c) & (d))) + +/* Rotates x right n bits. */ +#define ROTR(x, n) \ + (((x) >> (n)) | ((x) << ((sizeof (x) * NBBY)-(n)))) + +/* Shift x right n bits */ +#define SHR(x, n) ((x) >> (n)) + +/* SHA256 Functions */ +#define BIGSIGMA0_256(x) (ROTR((x), 2) ^ ROTR((x), 13) ^ ROTR((x), 22)) +#define BIGSIGMA1_256(x) (ROTR((x), 6) ^ ROTR((x), 11) ^ ROTR((x), 25)) +#define SIGMA0_256(x) (ROTR((x), 7) ^ ROTR((x), 18) ^ SHR((x), 3)) +#define SIGMA1_256(x) (ROTR((x), 17) ^ ROTR((x), 19) ^ SHR((x), 10)) + +#define SHA256ROUND(a, b, c, d, e, f, g, h, i, w) \ + T1 = h + BIGSIGMA1_256(e) + Ch(e, f, g) + SHA256_CONST(i) + w; \ + d += T1; \ + T2 = BIGSIGMA0_256(a) + Maj(a, b, c); \ + h = T1 + T2 + +/* + * sparc optimization: + * + * on the sparc, we can load big endian 32-bit data easily. note that + * special care must be taken to ensure the address is 32-bit aligned. + * in the interest of speed, we don't check to make sure, since + * careful programming can guarantee this for us. + */ + +#if defined(_BIG_ENDIAN) +#define LOAD_BIG_32(addr) (*(uint32_t *)(addr)) +#define LOAD_BIG_64(addr) (*(uint64_t *)(addr)) + +#elif defined(HAVE_HTONL) +#define LOAD_BIG_32(addr) htonl(*((uint32_t *)(addr))) +#define LOAD_BIG_64(addr) htonll(*((uint64_t *)(addr))) + +#else +/* little endian -- will work on big endian, but slowly */ +#define LOAD_BIG_32(addr) \ + (((addr)[0] << 24) | ((addr)[1] << 16) | ((addr)[2] << 8) | (addr)[3]) +#define LOAD_BIG_64(addr) \ + (((uint64_t)(addr)[0] << 56) | ((uint64_t)(addr)[1] << 48) | \ + ((uint64_t)(addr)[2] << 40) | ((uint64_t)(addr)[3] << 32) | \ + ((uint64_t)(addr)[4] << 24) | ((uint64_t)(addr)[5] << 16) | \ + ((uint64_t)(addr)[6] << 8) | (uint64_t)(addr)[7]) +#endif /* _BIG_ENDIAN */ + + +#if !defined(__amd64) +/* SHA256 Transform */ + +static void +SHA256Transform(SHA2_CTX *ctx, const uint8_t *blk) +{ + uint32_t a = ctx->state.s32[0]; + uint32_t b = ctx->state.s32[1]; + uint32_t c = ctx->state.s32[2]; + uint32_t d = ctx->state.s32[3]; + uint32_t e = ctx->state.s32[4]; + uint32_t f = ctx->state.s32[5]; + uint32_t g = ctx->state.s32[6]; + uint32_t h = ctx->state.s32[7]; + + uint32_t w0, w1, w2, w3, w4, w5, w6, w7; + uint32_t w8, w9, w10, w11, w12, w13, w14, w15; + uint32_t T1, T2; + + if ((uintptr_t)blk & 0x3) { /* not 4-byte aligned? */ + bcopy(blk, ctx->buf_un.buf32, sizeof (ctx->buf_un.buf32)); + blk = (uint8_t *)ctx->buf_un.buf32; + } + + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w0 = LOAD_BIG_32(blk + 4 * 0); + SHA256ROUND(a, b, c, d, e, f, g, h, 0, w0); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w1 = LOAD_BIG_32(blk + 4 * 1); + SHA256ROUND(h, a, b, c, d, e, f, g, 1, w1); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w2 = LOAD_BIG_32(blk + 4 * 2); + SHA256ROUND(g, h, a, b, c, d, e, f, 2, w2); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w3 = LOAD_BIG_32(blk + 4 * 3); + SHA256ROUND(f, g, h, a, b, c, d, e, 3, w3); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w4 = LOAD_BIG_32(blk + 4 * 4); + SHA256ROUND(e, f, g, h, a, b, c, d, 4, w4); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w5 = LOAD_BIG_32(blk + 4 * 5); + SHA256ROUND(d, e, f, g, h, a, b, c, 5, w5); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w6 = LOAD_BIG_32(blk + 4 * 6); + SHA256ROUND(c, d, e, f, g, h, a, b, 6, w6); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w7 = LOAD_BIG_32(blk + 4 * 7); + SHA256ROUND(b, c, d, e, f, g, h, a, 7, w7); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w8 = LOAD_BIG_32(blk + 4 * 8); + SHA256ROUND(a, b, c, d, e, f, g, h, 8, w8); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w9 = LOAD_BIG_32(blk + 4 * 9); + SHA256ROUND(h, a, b, c, d, e, f, g, 9, w9); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w10 = LOAD_BIG_32(blk + 4 * 10); + SHA256ROUND(g, h, a, b, c, d, e, f, 10, w10); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w11 = LOAD_BIG_32(blk + 4 * 11); + SHA256ROUND(f, g, h, a, b, c, d, e, 11, w11); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w12 = LOAD_BIG_32(blk + 4 * 12); + SHA256ROUND(e, f, g, h, a, b, c, d, 12, w12); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w13 = LOAD_BIG_32(blk + 4 * 13); + SHA256ROUND(d, e, f, g, h, a, b, c, 13, w13); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w14 = LOAD_BIG_32(blk + 4 * 14); + SHA256ROUND(c, d, e, f, g, h, a, b, 14, w14); + /* LINTED E_BAD_PTR_CAST_ALIGN */ + w15 = LOAD_BIG_32(blk + 4 * 15); + SHA256ROUND(b, c, d, e, f, g, h, a, 15, w15); + + w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0; + SHA256ROUND(a, b, c, d, e, f, g, h, 16, w0); + w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1; + SHA256ROUND(h, a, b, c, d, e, f, g, 17, w1); + w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2; + SHA256ROUND(g, h, a, b, c, d, e, f, 18, w2); + w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3; + SHA256ROUND(f, g, h, a, b, c, d, e, 19, w3); + w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4; + SHA256ROUND(e, f, g, h, a, b, c, d, 20, w4); + w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5; + SHA256ROUND(d, e, f, g, h, a, b, c, 21, w5); + w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6; + SHA256ROUND(c, d, e, f, g, h, a, b, 22, w6); + w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7; + SHA256ROUND(b, c, d, e, f, g, h, a, 23, w7); + w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8; + SHA256ROUND(a, b, c, d, e, f, g, h, 24, w8); + w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9; + SHA256ROUND(h, a, b, c, d, e, f, g, 25, w9); + w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10; + SHA256ROUND(g, h, a, b, c, d, e, f, 26, w10); + w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11; + SHA256ROUND(f, g, h, a, b, c, d, e, 27, w11); + w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12; + SHA256ROUND(e, f, g, h, a, b, c, d, 28, w12); + w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13; + SHA256ROUND(d, e, f, g, h, a, b, c, 29, w13); + w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14; + SHA256ROUND(c, d, e, f, g, h, a, b, 30, w14); + w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15; + SHA256ROUND(b, c, d, e, f, g, h, a, 31, w15); + + w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0; + SHA256ROUND(a, b, c, d, e, f, g, h, 32, w0); + w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1; + SHA256ROUND(h, a, b, c, d, e, f, g, 33, w1); + w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2; + SHA256ROUND(g, h, a, b, c, d, e, f, 34, w2); + w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3; + SHA256ROUND(f, g, h, a, b, c, d, e, 35, w3); + w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4; + SHA256ROUND(e, f, g, h, a, b, c, d, 36, w4); + w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5; + SHA256ROUND(d, e, f, g, h, a, b, c, 37, w5); + w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6; + SHA256ROUND(c, d, e, f, g, h, a, b, 38, w6); + w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7; + SHA256ROUND(b, c, d, e, f, g, h, a, 39, w7); + w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8; + SHA256ROUND(a, b, c, d, e, f, g, h, 40, w8); + w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9; + SHA256ROUND(h, a, b, c, d, e, f, g, 41, w9); + w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10; + SHA256ROUND(g, h, a, b, c, d, e, f, 42, w10); + w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11; + SHA256ROUND(f, g, h, a, b, c, d, e, 43, w11); + w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12; + SHA256ROUND(e, f, g, h, a, b, c, d, 44, w12); + w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13; + SHA256ROUND(d, e, f, g, h, a, b, c, 45, w13); + w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14; + SHA256ROUND(c, d, e, f, g, h, a, b, 46, w14); + w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15; + SHA256ROUND(b, c, d, e, f, g, h, a, 47, w15); + + w0 = SIGMA1_256(w14) + w9 + SIGMA0_256(w1) + w0; + SHA256ROUND(a, b, c, d, e, f, g, h, 48, w0); + w1 = SIGMA1_256(w15) + w10 + SIGMA0_256(w2) + w1; + SHA256ROUND(h, a, b, c, d, e, f, g, 49, w1); + w2 = SIGMA1_256(w0) + w11 + SIGMA0_256(w3) + w2; + SHA256ROUND(g, h, a, b, c, d, e, f, 50, w2); + w3 = SIGMA1_256(w1) + w12 + SIGMA0_256(w4) + w3; + SHA256ROUND(f, g, h, a, b, c, d, e, 51, w3); + w4 = SIGMA1_256(w2) + w13 + SIGMA0_256(w5) + w4; + SHA256ROUND(e, f, g, h, a, b, c, d, 52, w4); + w5 = SIGMA1_256(w3) + w14 + SIGMA0_256(w6) + w5; + SHA256ROUND(d, e, f, g, h, a, b, c, 53, w5); + w6 = SIGMA1_256(w4) + w15 + SIGMA0_256(w7) + w6; + SHA256ROUND(c, d, e, f, g, h, a, b, 54, w6); + w7 = SIGMA1_256(w5) + w0 + SIGMA0_256(w8) + w7; + SHA256ROUND(b, c, d, e, f, g, h, a, 55, w7); + w8 = SIGMA1_256(w6) + w1 + SIGMA0_256(w9) + w8; + SHA256ROUND(a, b, c, d, e, f, g, h, 56, w8); + w9 = SIGMA1_256(w7) + w2 + SIGMA0_256(w10) + w9; + SHA256ROUND(h, a, b, c, d, e, f, g, 57, w9); + w10 = SIGMA1_256(w8) + w3 + SIGMA0_256(w11) + w10; + SHA256ROUND(g, h, a, b, c, d, e, f, 58, w10); + w11 = SIGMA1_256(w9) + w4 + SIGMA0_256(w12) + w11; + SHA256ROUND(f, g, h, a, b, c, d, e, 59, w11); + w12 = SIGMA1_256(w10) + w5 + SIGMA0_256(w13) + w12; + SHA256ROUND(e, f, g, h, a, b, c, d, 60, w12); + w13 = SIGMA1_256(w11) + w6 + SIGMA0_256(w14) + w13; + SHA256ROUND(d, e, f, g, h, a, b, c, 61, w13); + w14 = SIGMA1_256(w12) + w7 + SIGMA0_256(w15) + w14; + SHA256ROUND(c, d, e, f, g, h, a, b, 62, w14); + w15 = SIGMA1_256(w13) + w8 + SIGMA0_256(w0) + w15; + SHA256ROUND(b, c, d, e, f, g, h, a, 63, w15); + + ctx->state.s32[0] += a; + ctx->state.s32[1] += b; + ctx->state.s32[2] += c; + ctx->state.s32[3] += d; + ctx->state.s32[4] += e; + ctx->state.s32[5] += f; + ctx->state.s32[6] += g; + ctx->state.s32[7] += h; +} +#endif /* !__amd64 */ + + +/* + * Encode() + * + * purpose: to convert a list of numbers from little endian to big endian + * input: uint8_t * : place to store the converted big endian numbers + * uint32_t * : place to get numbers to convert from + * size_t : the length of the input in bytes + * output: void + */ + +static void +Encode(uint8_t *_RESTRICT_KYWD output, uint32_t *_RESTRICT_KYWD input, + size_t len) +{ + size_t i, j; + + for (i = 0, j = 0; j < len; i++, j += 4) { + output[j] = (input[i] >> 24) & 0xff; + output[j + 1] = (input[i] >> 16) & 0xff; + output[j + 2] = (input[i] >> 8) & 0xff; + output[j + 3] = input[i] & 0xff; + } +} + +void +SHA2Init(uint64_t mech, SHA2_CTX *ctx) +{ + + switch (mech) { + case SHA256_MECH_INFO_TYPE: + case SHA256_HMAC_MECH_INFO_TYPE: + case SHA256_HMAC_GEN_MECH_INFO_TYPE: + ctx->state.s32[0] = 0x6a09e667U; + ctx->state.s32[1] = 0xbb67ae85U; + ctx->state.s32[2] = 0x3c6ef372U; + ctx->state.s32[3] = 0xa54ff53aU; + ctx->state.s32[4] = 0x510e527fU; + ctx->state.s32[5] = 0x9b05688cU; + ctx->state.s32[6] = 0x1f83d9abU; + ctx->state.s32[7] = 0x5be0cd19U; + break; + default: + cmn_err(CE_PANIC, + "sha2_init: failed to find a supported algorithm: 0x%x", + (uint32_t)mech); + } + + ctx->algotype = (uint32_t)mech; + ctx->count.c64[0] = ctx->count.c64[1] = 0; +} + +void +SHA256Init(SHA256_CTX *ctx) +{ + SHA2Init(SHA256, ctx); +} + +/* + * SHA2Update() + * + * purpose: continues an sha2 digest operation, using the message block + * to update the context. + * input: SHA2_CTX * : the context to update + * void * : the message block + * size_t : the length of the message block, in bytes + * output: void + */ + +void +SHA2Update(SHA2_CTX *ctx, const void *inptr, size_t input_len) +{ + uint32_t i, buf_index, buf_len, buf_limit; + const uint8_t *input = inptr; + uint32_t algotype = ctx->algotype; +#if defined(__amd64) + uint32_t block_count; +#endif /* !__amd64 */ + + + /* check for noop */ + if (input_len == 0) + return; + + if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) { + buf_limit = 64; + + /* compute number of bytes mod 64 */ + buf_index = (ctx->count.c32[1] >> 3) & 0x3F; + + /* update number of bits */ + if ((ctx->count.c32[1] += (input_len << 3)) < (input_len << 3)) + ctx->count.c32[0]++; + + ctx->count.c32[0] += (input_len >> 29); + + } else { + buf_limit = 128; + + /* compute number of bytes mod 128 */ + buf_index = (ctx->count.c64[1] >> 3) & 0x7F; + + /* update number of bits */ + if ((ctx->count.c64[1] += (input_len << 3)) < (input_len << 3)) + ctx->count.c64[0]++; + + ctx->count.c64[0] += (input_len >> 29); + } + + buf_len = buf_limit - buf_index; + + /* transform as many times as possible */ + i = 0; + if (input_len >= buf_len) { + + /* + * general optimization: + * + * only do initial bcopy() and SHA2Transform() if + * buf_index != 0. if buf_index == 0, we're just + * wasting our time doing the bcopy() since there + * wasn't any data left over from a previous call to + * SHA2Update(). + */ + if (buf_index) { + bcopy(input, &ctx->buf_un.buf8[buf_index], buf_len); + if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) + SHA256Transform(ctx, ctx->buf_un.buf8); + + i = buf_len; + } + +#if !defined(__amd64) + if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) { + for (; i + buf_limit - 1 < input_len; i += buf_limit) { + SHA256Transform(ctx, &input[i]); + } + } + +#else + if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) { + block_count = (input_len - i) >> 6; + if (block_count > 0) { + SHA256TransformBlocks(ctx, &input[i], + block_count); + i += block_count << 6; + } + } +#endif /* !__amd64 */ + + /* + * general optimization: + * + * if i and input_len are the same, return now instead + * of calling bcopy(), since the bcopy() in this case + * will be an expensive noop. + */ + + if (input_len == i) + return; + + buf_index = 0; + } + + /* buffer remaining input */ + bcopy(&input[i], &ctx->buf_un.buf8[buf_index], input_len - i); +} + + +/* + * SHA2Final() + * + * purpose: ends an sha2 digest operation, finalizing the message digest and + * zeroing the context. + * input: uchar_t * : a buffer to store the digest + * : The function actually uses void* because many + * : callers pass things other than uchar_t here. + * SHA2_CTX * : the context to finalize, save, and zero + * output: void + */ + +void +SHA2Final(void *digest, SHA2_CTX *ctx) +{ + uint8_t bitcount_be[sizeof (ctx->count.c32)]; + uint32_t index; + uint32_t algotype = ctx->algotype; + + if (algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) { + index = (ctx->count.c32[1] >> 3) & 0x3f; + Encode(bitcount_be, ctx->count.c32, sizeof (bitcount_be)); + SHA2Update(ctx, PADDING, ((index < 56) ? 56 : 120) - index); + SHA2Update(ctx, bitcount_be, sizeof (bitcount_be)); + Encode(digest, ctx->state.s32, sizeof (ctx->state.s32)); + } + + /* zeroize sensitive information */ + bzero(ctx, sizeof (*ctx)); +} diff --git a/module/icp/api/kcf_cipher.c b/module/icp/api/kcf_cipher.c new file mode 100644 index 000000000000..2585b7fedae7 --- /dev/null +++ b/module/icp/api/kcf_cipher.c @@ -0,0 +1,935 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include +#include + +#define CRYPTO_OPS_OFFSET(f) offsetof(crypto_ops_t, co_##f) +#define CRYPTO_CIPHER_OFFSET(f) offsetof(crypto_cipher_ops_t, f) + +/* + * Encryption and decryption routines. + */ + +/* + * The following are the possible returned values common to all the routines + * below. The applicability of some of these return values depends on the + * presence of the arguments. + * + * CRYPTO_SUCCESS: The operation completed successfully. + * CRYPTO_QUEUED: A request was submitted successfully. The callback + * routine will be called when the operation is done. + * CRYPTO_INVALID_MECH_NUMBER, CRYPTO_INVALID_MECH_PARAM, or + * CRYPTO_INVALID_MECH for problems with the 'mech'. + * CRYPTO_INVALID_DATA for bogus 'data' + * CRYPTO_HOST_MEMORY for failure to allocate memory to handle this work. + * CRYPTO_INVALID_CONTEXT: Not a valid context. + * CRYPTO_BUSY: Cannot process the request now. Schedule a + * crypto_bufcall(), or try later. + * CRYPTO_NOT_SUPPORTED and CRYPTO_MECH_NOT_SUPPORTED: No provider is + * capable of a function or a mechanism. + * CRYPTO_INVALID_KEY: bogus 'key' argument. + * CRYPTO_INVALID_PLAINTEXT: bogus 'plaintext' argument. + * CRYPTO_INVALID_CIPHERTEXT: bogus 'ciphertext' argument. + */ + +/* + * crypto_cipher_init_prov() + * + * Arguments: + * + * pd: provider descriptor + * sid: session id + * mech: crypto_mechanism_t pointer. + * mech_type is a valid value previously returned by + * crypto_mech2id(); + * When the mech's parameter is not NULL, its definition depends + * on the standard definition of the mechanism. + * key: pointer to a crypto_key_t structure. + * tmpl: a crypto_ctx_template_t, opaque template of a context of an + * encryption or decryption with the 'mech' using 'key'. + * 'tmpl' is created by a previous call to + * crypto_create_ctx_template(). + * ctxp: Pointer to a crypto_context_t. + * func: CRYPTO_FG_ENCRYPT or CRYPTO_FG_DECRYPT. + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * This is a common function invoked internally by both + * crypto_encrypt_init() and crypto_decrypt_init(). + * Asynchronously submits a request for, or synchronously performs the + * initialization of an encryption or a decryption operation. + * When possible and applicable, will internally use the pre-expanded key + * schedule from the context template, tmpl. + * When complete and successful, 'ctxp' will contain a crypto_context_t + * valid for later calls to encrypt_update() and encrypt_final(), or + * decrypt_update() and decrypt_final(). + * The caller should hold a reference on the specified provider + * descriptor before calling this function. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +static int +crypto_cipher_init_prov(crypto_provider_t provider, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_key_t *key, + crypto_spi_ctx_template_t tmpl, crypto_context_t *ctxp, + crypto_call_req_t *crq, crypto_func_group_t func) +{ + int error; + crypto_ctx_t *ctx; + kcf_req_params_t params; + kcf_provider_desc_t *pd = provider; + kcf_provider_desc_t *real_provider = pd; + + ASSERT(KCF_PROV_REFHELD(pd)); + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + if (func == CRYPTO_FG_ENCRYPT) { + error = kcf_get_hardware_provider(mech->cm_type, + CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd, + &real_provider, CRYPTO_FG_ENCRYPT); + } else { + error = kcf_get_hardware_provider(mech->cm_type, + CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd, + &real_provider, CRYPTO_FG_DECRYPT); + } + + if (error != CRYPTO_SUCCESS) + return (error); + } + + /* Allocate and initialize the canonical context */ + if ((ctx = kcf_new_ctx(crq, real_provider, sid)) == NULL) { + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + return (CRYPTO_HOST_MEMORY); + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(crq, pd)) { + crypto_mechanism_t lmech; + + lmech = *mech; + KCF_SET_PROVIDER_MECHNUM(mech->cm_type, real_provider, &lmech); + + if (func == CRYPTO_FG_ENCRYPT) + error = KCF_PROV_ENCRYPT_INIT(real_provider, ctx, + &lmech, key, tmpl, KCF_SWFP_RHNDL(crq)); + else { + ASSERT(func == CRYPTO_FG_DECRYPT); + + error = KCF_PROV_DECRYPT_INIT(real_provider, ctx, + &lmech, key, tmpl, KCF_SWFP_RHNDL(crq)); + } + KCF_PROV_INCRSTATS(pd, error); + + goto done; + } + + /* Check if context sharing is possible */ + if (pd->pd_prov_type == CRYPTO_HW_PROVIDER && + key->ck_format == CRYPTO_KEY_RAW && + KCF_CAN_SHARE_OPSTATE(pd, mech->cm_type)) { + kcf_context_t *tctxp = (kcf_context_t *)ctx; + kcf_provider_desc_t *tpd = NULL; + crypto_mech_info_t *sinfo; + + if ((kcf_get_sw_prov(mech->cm_type, &tpd, &tctxp->kc_mech, + B_FALSE) == CRYPTO_SUCCESS)) { + int tlen; + + sinfo = &(KCF_TO_PROV_MECHINFO(tpd, mech->cm_type)); + /* + * key->ck_length from the consumer is always in bits. + * We convert it to be in the same unit registered by + * the provider in order to do a comparison. + */ + if (sinfo->cm_mech_flags & CRYPTO_KEYSIZE_UNIT_IN_BYTES) + tlen = key->ck_length >> 3; + else + tlen = key->ck_length; + /* + * Check if the software provider can support context + * sharing and support this key length. + */ + if ((sinfo->cm_mech_flags & CRYPTO_CAN_SHARE_OPSTATE) && + (tlen >= sinfo->cm_min_key_length) && + (tlen <= sinfo->cm_max_key_length)) { + ctx->cc_flags = CRYPTO_INIT_OPSTATE; + tctxp->kc_sw_prov_desc = tpd; + } else + KCF_PROV_REFRELE(tpd); + } + } + + if (func == CRYPTO_FG_ENCRYPT) { + KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_INIT, sid, + mech, key, NULL, NULL, tmpl); + } else { + ASSERT(func == CRYPTO_FG_DECRYPT); + KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_INIT, sid, + mech, key, NULL, NULL, tmpl); + } + + error = kcf_submit_request(real_provider, ctx, crq, ¶ms, + B_FALSE); + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + +done: + if ((error == CRYPTO_SUCCESS) || (error == CRYPTO_QUEUED)) + *ctxp = (crypto_context_t)ctx; + else { + /* Release the hold done in kcf_new_ctx(). */ + KCF_CONTEXT_REFRELE((kcf_context_t *)ctx->cc_framework_private); + } + + return (error); +} + +/* + * Same as crypto_cipher_init_prov(), but relies on the scheduler to pick + * an appropriate provider. See crypto_cipher_init_prov() comments for more + * details. + */ +static int +crypto_cipher_init(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, + crypto_call_req_t *crq, crypto_func_group_t func) +{ + int error; + kcf_mech_entry_t *me; + kcf_provider_desc_t *pd; + kcf_ctx_template_t *ctx_tmpl; + crypto_spi_ctx_template_t spi_ctx_tmpl = NULL; + kcf_prov_tried_t *list = NULL; + +retry: + /* pd is returned held */ + if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error, + list, func, CHECK_RESTRICT(crq), 0)) == NULL) { + if (list != NULL) + kcf_free_triedlist(list); + return (error); + } + + /* + * For SW providers, check the validity of the context template + * It is very rare that the generation number mis-matches, so + * is acceptable to fail here, and let the consumer recover by + * freeing this tmpl and create a new one for the key and new SW + * provider + */ + if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && + ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) { + if (ctx_tmpl->ct_generation != me->me_gen_swprov) { + if (list != NULL) + kcf_free_triedlist(list); + KCF_PROV_REFRELE(pd); + return (CRYPTO_OLD_CTX_TEMPLATE); + } else { + spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl; + } + } + + error = crypto_cipher_init_prov(pd, pd->pd_sid, mech, key, + spi_ctx_tmpl, ctxp, crq, func); + if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED && + IS_RECOVERABLE(error)) { + /* Add pd to the linked list of providers tried. */ + if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL) + goto retry; + } + + if (list != NULL) + kcf_free_triedlist(list); + + KCF_PROV_REFRELE(pd); + return (error); +} + +/* + * crypto_encrypt_prov() + * + * Arguments: + * pd: provider descriptor + * sid: session id + * mech: crypto_mechanism_t pointer. + * mech_type is a valid value previously returned by + * crypto_mech2id(); + * When the mech's parameter is not NULL, its definition depends + * on the standard definition of the mechanism. + * key: pointer to a crypto_key_t structure. + * plaintext: The message to be encrypted + * ciphertext: Storage for the encrypted message. The length needed + * depends on the mechanism, and the plaintext's size. + * tmpl: a crypto_ctx_template_t, opaque template of a context of an + * encryption with the 'mech' using 'key'. 'tmpl' is created by + * a previous call to crypto_create_ctx_template(). + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs a + * single-part encryption of 'plaintext' with the mechanism 'mech', using + * the key 'key'. + * When complete and successful, 'ciphertext' will contain the encrypted + * message. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_encrypt_prov(crypto_provider_t provider, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_data_t *plaintext, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_data_t *ciphertext, + crypto_call_req_t *crq) +{ + kcf_req_params_t params; + kcf_provider_desc_t *pd = provider; + kcf_provider_desc_t *real_provider = pd; + int error; + + ASSERT(KCF_PROV_REFHELD(pd)); + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + error = kcf_get_hardware_provider(mech->cm_type, + CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd, + &real_provider, CRYPTO_FG_ENCRYPT_ATOMIC); + + if (error != CRYPTO_SUCCESS) + return (error); + } + + KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, sid, mech, key, + plaintext, ciphertext, tmpl); + + error = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE); + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + + return (error); +} + +/* + * Same as crypto_encrypt_prov(), but relies on the scheduler to pick + * a provider. See crypto_encrypt_prov() for more details. + */ +int +crypto_encrypt(crypto_mechanism_t *mech, crypto_data_t *plaintext, + crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *ciphertext, + crypto_call_req_t *crq) +{ + int error; + kcf_mech_entry_t *me; + kcf_req_params_t params; + kcf_provider_desc_t *pd; + kcf_ctx_template_t *ctx_tmpl; + crypto_spi_ctx_template_t spi_ctx_tmpl = NULL; + kcf_prov_tried_t *list = NULL; + +retry: + /* pd is returned held */ + if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error, + list, CRYPTO_FG_ENCRYPT_ATOMIC, CHECK_RESTRICT(crq), + plaintext->cd_length)) == NULL) { + if (list != NULL) + kcf_free_triedlist(list); + return (error); + } + + /* + * For SW providers, check the validity of the context template + * It is very rare that the generation number mis-matches, so + * is acceptable to fail here, and let the consumer recover by + * freeing this tmpl and create a new one for the key and new SW + * provider + */ + if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && + ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) { + if (ctx_tmpl->ct_generation != me->me_gen_swprov) { + if (list != NULL) + kcf_free_triedlist(list); + KCF_PROV_REFRELE(pd); + return (CRYPTO_OLD_CTX_TEMPLATE); + } else { + spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl; + } + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(crq, pd)) { + crypto_mechanism_t lmech; + + lmech = *mech; + KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech); + + error = KCF_PROV_ENCRYPT_ATOMIC(pd, pd->pd_sid, &lmech, key, + plaintext, ciphertext, spi_ctx_tmpl, KCF_SWFP_RHNDL(crq)); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, pd->pd_sid, + mech, key, plaintext, ciphertext, spi_ctx_tmpl); + error = kcf_submit_request(pd, NULL, crq, ¶ms, B_FALSE); + } + + if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED && + IS_RECOVERABLE(error)) { + /* Add pd to the linked list of providers tried. */ + if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL) + goto retry; + } + + if (list != NULL) + kcf_free_triedlist(list); + + KCF_PROV_REFRELE(pd); + return (error); +} + +/* + * crypto_encrypt_init_prov() + * + * Calls crypto_cipher_init_prov() to initialize an encryption operation. + */ +int +crypto_encrypt_init_prov(crypto_provider_t pd, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, + crypto_call_req_t *crq) +{ + return (crypto_cipher_init_prov(pd, sid, mech, key, tmpl, ctxp, crq, + CRYPTO_FG_ENCRYPT)); +} + +/* + * crypto_encrypt_init() + * + * Calls crypto_cipher_init() to initialize an encryption operation + */ +int +crypto_encrypt_init(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, + crypto_call_req_t *crq) +{ + return (crypto_cipher_init(mech, key, tmpl, ctxp, crq, + CRYPTO_FG_ENCRYPT)); +} + +/* + * crypto_encrypt_update() + * + * Arguments: + * context: A crypto_context_t initialized by encrypt_init(). + * plaintext: The message part to be encrypted + * ciphertext: Storage for the encrypted message part. + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs a + * part of an encryption operation. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_encrypt_update(crypto_context_t context, crypto_data_t *plaintext, + crypto_data_t *ciphertext, crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_ENCRYPT_UPDATE(pd, ctx, plaintext, + ciphertext, NULL); + KCF_PROV_INCRSTATS(pd, error); + return (error); + } + + /* Check if we should use a software provider for small jobs */ + if ((ctx->cc_flags & CRYPTO_USE_OPSTATE) && cr == NULL) { + if (plaintext->cd_length < kcf_ctx->kc_mech->me_threshold && + kcf_ctx->kc_sw_prov_desc != NULL && + KCF_IS_PROV_USABLE(kcf_ctx->kc_sw_prov_desc)) { + pd = kcf_ctx->kc_sw_prov_desc; + } + } + + KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_UPDATE, + ctx->cc_session, NULL, NULL, plaintext, ciphertext, NULL); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + + return (error); +} + +/* + * crypto_encrypt_final() + * + * Arguments: + * context: A crypto_context_t initialized by encrypt_init(). + * ciphertext: Storage for the last part of encrypted message + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs the + * final part of an encryption operation. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_encrypt_final(crypto_context_t context, crypto_data_t *ciphertext, + crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_ENCRYPT_FINAL(pd, ctx, ciphertext, NULL); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_FINAL, + ctx->cc_session, NULL, NULL, NULL, ciphertext, NULL); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + /* Release the hold done in kcf_new_ctx() during init step. */ + KCF_CONTEXT_COND_RELEASE(error, kcf_ctx); + return (error); +} + +/* + * crypto_decrypt_prov() + * + * Arguments: + * pd: provider descriptor + * sid: session id + * mech: crypto_mechanism_t pointer. + * mech_type is a valid value previously returned by + * crypto_mech2id(); + * When the mech's parameter is not NULL, its definition depends + * on the standard definition of the mechanism. + * key: pointer to a crypto_key_t structure. + * ciphertext: The message to be encrypted + * plaintext: Storage for the encrypted message. The length needed + * depends on the mechanism, and the plaintext's size. + * tmpl: a crypto_ctx_template_t, opaque template of a context of an + * encryption with the 'mech' using 'key'. 'tmpl' is created by + * a previous call to crypto_create_ctx_template(). + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs a + * single-part decryption of 'ciphertext' with the mechanism 'mech', using + * the key 'key'. + * When complete and successful, 'plaintext' will contain the decrypted + * message. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_decrypt_prov(crypto_provider_t provider, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_data_t *ciphertext, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_data_t *plaintext, + crypto_call_req_t *crq) +{ + kcf_req_params_t params; + kcf_provider_desc_t *pd = provider; + kcf_provider_desc_t *real_provider = pd; + int rv; + + ASSERT(KCF_PROV_REFHELD(pd)); + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + rv = kcf_get_hardware_provider(mech->cm_type, + CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd, + &real_provider, CRYPTO_FG_DECRYPT_ATOMIC); + + if (rv != CRYPTO_SUCCESS) + return (rv); + } + + KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, sid, mech, key, + ciphertext, plaintext, tmpl); + + rv = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE); + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + + return (rv); +} + +/* + * Same as crypto_decrypt_prov(), but relies on the KCF scheduler to + * choose a provider. See crypto_decrypt_prov() comments for more + * information. + */ +int +crypto_decrypt(crypto_mechanism_t *mech, crypto_data_t *ciphertext, + crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *plaintext, + crypto_call_req_t *crq) +{ + int error; + kcf_mech_entry_t *me; + kcf_req_params_t params; + kcf_provider_desc_t *pd; + kcf_ctx_template_t *ctx_tmpl; + crypto_spi_ctx_template_t spi_ctx_tmpl = NULL; + kcf_prov_tried_t *list = NULL; + +retry: + /* pd is returned held */ + if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error, + list, CRYPTO_FG_DECRYPT_ATOMIC, CHECK_RESTRICT(crq), + ciphertext->cd_length)) == NULL) { + if (list != NULL) + kcf_free_triedlist(list); + return (error); + } + + /* + * For SW providers, check the validity of the context template + * It is very rare that the generation number mis-matches, so + * is acceptable to fail here, and let the consumer recover by + * freeing this tmpl and create a new one for the key and new SW + * provider + */ + if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && + ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) { + if (ctx_tmpl->ct_generation != me->me_gen_swprov) { + if (list != NULL) + kcf_free_triedlist(list); + KCF_PROV_REFRELE(pd); + return (CRYPTO_OLD_CTX_TEMPLATE); + } else { + spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl; + } + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(crq, pd)) { + crypto_mechanism_t lmech; + + lmech = *mech; + KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech); + + error = KCF_PROV_DECRYPT_ATOMIC(pd, pd->pd_sid, &lmech, key, + ciphertext, plaintext, spi_ctx_tmpl, KCF_SWFP_RHNDL(crq)); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, pd->pd_sid, + mech, key, ciphertext, plaintext, spi_ctx_tmpl); + error = kcf_submit_request(pd, NULL, crq, ¶ms, B_FALSE); + } + + if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED && + IS_RECOVERABLE(error)) { + /* Add pd to the linked list of providers tried. */ + if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL) + goto retry; + } + + if (list != NULL) + kcf_free_triedlist(list); + + KCF_PROV_REFRELE(pd); + return (error); +} + +/* + * crypto_decrypt_init_prov() + * + * Calls crypto_cipher_init_prov() to initialize a decryption operation + */ +int +crypto_decrypt_init_prov(crypto_provider_t pd, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, + crypto_call_req_t *crq) +{ + return (crypto_cipher_init_prov(pd, sid, mech, key, tmpl, ctxp, crq, + CRYPTO_FG_DECRYPT)); +} + +/* + * crypto_decrypt_init() + * + * Calls crypto_cipher_init() to initialize a decryption operation + */ +int +crypto_decrypt_init(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, + crypto_call_req_t *crq) +{ + return (crypto_cipher_init(mech, key, tmpl, ctxp, crq, + CRYPTO_FG_DECRYPT)); +} + +/* + * crypto_decrypt_update() + * + * Arguments: + * context: A crypto_context_t initialized by decrypt_init(). + * ciphertext: The message part to be decrypted + * plaintext: Storage for the decrypted message part. + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs a + * part of an decryption operation. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_decrypt_update(crypto_context_t context, crypto_data_t *ciphertext, + crypto_data_t *plaintext, crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_DECRYPT_UPDATE(pd, ctx, ciphertext, + plaintext, NULL); + KCF_PROV_INCRSTATS(pd, error); + return (error); + } + + /* Check if we should use a software provider for small jobs */ + if ((ctx->cc_flags & CRYPTO_USE_OPSTATE) && cr == NULL) { + if (ciphertext->cd_length < kcf_ctx->kc_mech->me_threshold && + kcf_ctx->kc_sw_prov_desc != NULL && + KCF_IS_PROV_USABLE(kcf_ctx->kc_sw_prov_desc)) { + pd = kcf_ctx->kc_sw_prov_desc; + } + } + + KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_UPDATE, + ctx->cc_session, NULL, NULL, ciphertext, plaintext, NULL); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + + return (error); +} + +/* + * crypto_decrypt_final() + * + * Arguments: + * context: A crypto_context_t initialized by decrypt_init(). + * plaintext: Storage for the last part of the decrypted message + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs the + * final part of a decryption operation. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_decrypt_final(crypto_context_t context, crypto_data_t *plaintext, + crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_DECRYPT_FINAL(pd, ctx, plaintext, + NULL); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_FINAL, + ctx->cc_session, NULL, NULL, NULL, plaintext, NULL); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + /* Release the hold done in kcf_new_ctx() during init step. */ + KCF_CONTEXT_COND_RELEASE(error, kcf_ctx); + return (error); +} + +/* + * See comments for crypto_encrypt_update(). + */ +int +crypto_encrypt_single(crypto_context_t context, crypto_data_t *plaintext, + crypto_data_t *ciphertext, crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_ENCRYPT(pd, ctx, plaintext, + ciphertext, NULL); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_ENCRYPT_OPS_PARAMS(¶ms, KCF_OP_SINGLE, pd->pd_sid, + NULL, NULL, plaintext, ciphertext, NULL); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + /* Release the hold done in kcf_new_ctx() during init step. */ + KCF_CONTEXT_COND_RELEASE(error, kcf_ctx); + return (error); +} + +/* + * See comments for crypto_decrypt_update(). + */ +int +crypto_decrypt_single(crypto_context_t context, crypto_data_t *ciphertext, + crypto_data_t *plaintext, crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_DECRYPT(pd, ctx, ciphertext, + plaintext, NULL); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_DECRYPT_OPS_PARAMS(¶ms, KCF_OP_SINGLE, pd->pd_sid, + NULL, NULL, ciphertext, plaintext, NULL); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + /* Release the hold done in kcf_new_ctx() during init step. */ + KCF_CONTEXT_COND_RELEASE(error, kcf_ctx); + return (error); +} + +#if defined(_KERNEL) && defined(HAVE_SPL) +EXPORT_SYMBOL(crypto_cipher_init_prov); +EXPORT_SYMBOL(crypto_cipher_init); +EXPORT_SYMBOL(crypto_encrypt_prov); +EXPORT_SYMBOL(crypto_encrypt); +EXPORT_SYMBOL(crypto_encrypt_init_prov); +EXPORT_SYMBOL(crypto_encrypt_init); +EXPORT_SYMBOL(crypto_encrypt_update); +EXPORT_SYMBOL(crypto_encrypt_final); +EXPORT_SYMBOL(crypto_decrypt_prov); +EXPORT_SYMBOL(crypto_decrypt); +EXPORT_SYMBOL(crypto_decrypt_init_prov); +EXPORT_SYMBOL(crypto_decrypt_init); +EXPORT_SYMBOL(crypto_decrypt_update); +EXPORT_SYMBOL(crypto_decrypt_final); +EXPORT_SYMBOL(crypto_encrypt_single); +EXPORT_SYMBOL(crypto_decrypt_single); +#endif diff --git a/module/icp/api/kcf_ctxops.c b/module/icp/api/kcf_ctxops.c new file mode 100644 index 000000000000..3f90674b0a33 --- /dev/null +++ b/module/icp/api/kcf_ctxops.c @@ -0,0 +1,151 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include +#include + +/* + * Crypto contexts manipulation routines + */ + +/* + * crypto_create_ctx_template() + * + * Arguments: + * + * mech: crypto_mechanism_t pointer. + * mech_type is a valid value previously returned by + * crypto_mech2id(); + * When the mech's parameter is not NULL, its definition depends + * on the standard definition of the mechanism. + * key: pointer to a crypto_key_t structure. + * ptmpl: a storage for the opaque crypto_ctx_template_t, allocated and + * initialized by the software provider this routine is + * dispatched to. + * kmflag: KM_SLEEP/KM_NOSLEEP mem. alloc. flag. + * + * Description: + * Redirects the call to the software provider of the specified + * mechanism. That provider will allocate and pre-compute/pre-expand + * the context template, reusable by later calls to crypto_xxx_init(). + * The size and address of that provider context template are stored + * in an internal structure, kcf_ctx_template_t. The address of that + * structure is given back to the caller in *ptmpl. + * + * Context: + * Process or interrupt. + * + * Returns: + * CRYPTO_SUCCESS when the context template is successfully created. + * CRYPTO_HOST_MEMEORY: mem alloc failure + * CRYPTO_ARGUMENTS_BAD: NULL storage for the ctx template. + * RYPTO_MECHANISM_INVALID: invalid mechanism 'mech'. + */ +int +crypto_create_ctx_template(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t *ptmpl, int kmflag) +{ + int error; + kcf_mech_entry_t *me; + kcf_provider_desc_t *pd; + kcf_ctx_template_t *ctx_tmpl; + crypto_mechanism_t prov_mech; + + /* A few args validation */ + + if (ptmpl == NULL) + return (CRYPTO_ARGUMENTS_BAD); + + if (mech == NULL) + return (CRYPTO_MECHANISM_INVALID); + + error = kcf_get_sw_prov(mech->cm_type, &pd, &me, B_TRUE); + if (error != CRYPTO_SUCCESS) + return (error); + + if ((ctx_tmpl = (kcf_ctx_template_t *)kmem_alloc( + sizeof (kcf_ctx_template_t), kmflag)) == NULL) { + KCF_PROV_REFRELE(pd); + return (CRYPTO_HOST_MEMORY); + } + + /* Pass a mechtype that the provider understands */ + prov_mech.cm_type = KCF_TO_PROV_MECHNUM(pd, mech->cm_type); + prov_mech.cm_param = mech->cm_param; + prov_mech.cm_param_len = mech->cm_param_len; + + error = KCF_PROV_CREATE_CTX_TEMPLATE(pd, &prov_mech, key, + &(ctx_tmpl->ct_prov_tmpl), &(ctx_tmpl->ct_size), KCF_RHNDL(kmflag)); + + if (error == CRYPTO_SUCCESS) { + ctx_tmpl->ct_generation = me->me_gen_swprov; + *ptmpl = ctx_tmpl; + } else { + kmem_free(ctx_tmpl, sizeof (kcf_ctx_template_t)); + } + KCF_PROV_REFRELE(pd); + + return (error); +} + +/* + * crypto_destroy_ctx_template() + * + * Arguments: + * + * tmpl: an opaque crypto_ctx_template_t previously created by + * crypto_create_ctx_template() + * + * Description: + * Frees the inbedded crypto_spi_ctx_template_t, then the + * kcf_ctx_template_t. + * + * Context: + * Process or interrupt. + * + */ +void +crypto_destroy_ctx_template(crypto_ctx_template_t tmpl) +{ + kcf_ctx_template_t *ctx_tmpl = (kcf_ctx_template_t *)tmpl; + + if (ctx_tmpl == NULL) + return; + + ASSERT(ctx_tmpl->ct_prov_tmpl != NULL); + + bzero(ctx_tmpl->ct_prov_tmpl, ctx_tmpl->ct_size); + kmem_free(ctx_tmpl->ct_prov_tmpl, ctx_tmpl->ct_size); + kmem_free(ctx_tmpl, sizeof (kcf_ctx_template_t)); +} + +#if defined(_KERNEL) && defined(HAVE_SPL) +EXPORT_SYMBOL(crypto_create_ctx_template); +EXPORT_SYMBOL(crypto_destroy_ctx_template); +#endif diff --git a/module/icp/api/kcf_digest.c b/module/icp/api/kcf_digest.c new file mode 100644 index 000000000000..b58d3b452829 --- /dev/null +++ b/module/icp/api/kcf_digest.c @@ -0,0 +1,494 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include +#include + +#define CRYPTO_OPS_OFFSET(f) offsetof(crypto_ops_t, co_##f) +#define CRYPTO_DIGEST_OFFSET(f) offsetof(crypto_digest_ops_t, f) + +/* + * Message digest routines + */ + +/* + * The following are the possible returned values common to all the routines + * below. The applicability of some of these return values depends on the + * presence of the arguments. + * + * CRYPTO_SUCCESS: The operation completed successfully. + * CRYPTO_QUEUED: A request was submitted successfully. The callback + * routine will be called when the operation is done. + * CRYPTO_MECHANISM_INVALID or CRYPTO_INVALID_MECH_PARAM + * for problems with the 'mech'. + * CRYPTO_INVALID_DATA for bogus 'data' + * CRYPTO_HOST_MEMORY for failure to allocate memory to handle this work. + * CRYPTO_INVALID_CONTEXT: Not a valid context. + * CRYPTO_BUSY: Cannot process the request now. Schedule a + * crypto_bufcall(), or try later. + * CRYPTO_NOT_SUPPORTED and CRYPTO_MECH_NOT_SUPPORTED: + * No provider is capable of a function or a mechanism. + */ + + +/* + * crypto_digest_prov() + * + * Arguments: + * pd: pointer to the descriptor of the provider to use for this + * operation. + * sid: provider session id. + * mech: crypto_mechanism_t pointer. + * mech_type is a valid value previously returned by + * crypto_mech2id(); + * When the mech's parameter is not NULL, its definition depends + * on the standard definition of the mechanism. + * data: The message to be digested. + * digest: Storage for the digest. The length needed depends on the + * mechanism. + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs the + * digesting operation of 'data' on the specified + * provider with the specified session. + * When complete and successful, 'digest' will contain the digest value. + * The caller should hold a reference on the specified provider + * descriptor before calling this function. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_digest_prov(crypto_provider_t provider, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_data_t *data, crypto_data_t *digest, + crypto_call_req_t *crq) +{ + kcf_req_params_t params; + kcf_provider_desc_t *pd = provider; + kcf_provider_desc_t *real_provider = pd; + int rv; + + ASSERT(KCF_PROV_REFHELD(pd)); + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + rv = kcf_get_hardware_provider(mech->cm_type, + CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), + pd, &real_provider, CRYPTO_FG_DIGEST_ATOMIC); + + if (rv != CRYPTO_SUCCESS) + return (rv); + } + KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, sid, mech, NULL, + data, digest); + + /* no crypto context to carry between multiple parts. */ + rv = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE); + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + + return (rv); +} + + +/* + * Same as crypto_digest_prov(), but relies on the KCF scheduler to + * choose a provider. See crypto_digest_prov() comments for more information. + */ +int +crypto_digest(crypto_mechanism_t *mech, crypto_data_t *data, + crypto_data_t *digest, crypto_call_req_t *crq) +{ + int error; + kcf_provider_desc_t *pd; + kcf_req_params_t params; + kcf_prov_tried_t *list = NULL; + +retry: + /* The pd is returned held */ + if ((pd = kcf_get_mech_provider(mech->cm_type, NULL, &error, list, + CRYPTO_FG_DIGEST_ATOMIC, CHECK_RESTRICT(crq), + data->cd_length)) == NULL) { + if (list != NULL) + kcf_free_triedlist(list); + return (error); + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(crq, pd)) { + crypto_mechanism_t lmech; + + lmech = *mech; + KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech); + error = KCF_PROV_DIGEST_ATOMIC(pd, pd->pd_sid, &lmech, data, + digest, KCF_SWFP_RHNDL(crq)); + KCF_PROV_INCRSTATS(pd, error); + } else { + if (pd->pd_prov_type == CRYPTO_HW_PROVIDER && + (pd->pd_flags & CRYPTO_HASH_NO_UPDATE) && + (data->cd_length > pd->pd_hash_limit)) { + error = CRYPTO_BUFFER_TOO_BIG; + } else { + KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, + pd->pd_sid, mech, NULL, data, digest); + + /* no crypto context to carry between multiple parts. */ + error = kcf_submit_request(pd, NULL, crq, ¶ms, + B_FALSE); + } + } + + if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED && + IS_RECOVERABLE(error)) { + /* Add pd to the linked list of providers tried. */ + if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL) + goto retry; + } + + if (list != NULL) + kcf_free_triedlist(list); + + KCF_PROV_REFRELE(pd); + return (error); +} + +/* + * crypto_digest_init_prov() + * + * pd: pointer to the descriptor of the provider to use for this + * operation. + * sid: provider session id. + * mech: crypto_mechanism_t pointer. + * mech_type is a valid value previously returned by + * crypto_mech2id(); + * When the mech's parameter is not NULL, its definition depends + * on the standard definition of the mechanism. + * ctxp: Pointer to a crypto_context_t. + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs the + * initialization of a message digest operation on the specified + * provider with the specified session. + * When complete and successful, 'ctxp' will contain a crypto_context_t + * valid for later calls to digest_update() and digest_final(). + * The caller should hold a reference on the specified provider + * descriptor before calling this function. + */ +int +crypto_digest_init_prov(crypto_provider_t provider, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_context_t *ctxp, crypto_call_req_t *crq) +{ + int error; + crypto_ctx_t *ctx; + kcf_req_params_t params; + kcf_provider_desc_t *pd = provider; + kcf_provider_desc_t *real_provider = pd; + + ASSERT(KCF_PROV_REFHELD(pd)); + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + error = kcf_get_hardware_provider(mech->cm_type, + CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd, + &real_provider, CRYPTO_FG_DIGEST); + + if (error != CRYPTO_SUCCESS) + return (error); + } + + /* Allocate and initialize the canonical context */ + if ((ctx = kcf_new_ctx(crq, real_provider, sid)) == NULL) { + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + return (CRYPTO_HOST_MEMORY); + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(crq, pd)) { + crypto_mechanism_t lmech; + + lmech = *mech; + KCF_SET_PROVIDER_MECHNUM(mech->cm_type, real_provider, &lmech); + error = KCF_PROV_DIGEST_INIT(real_provider, ctx, &lmech, + KCF_SWFP_RHNDL(crq)); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_INIT, sid, + mech, NULL, NULL, NULL); + error = kcf_submit_request(real_provider, ctx, crq, ¶ms, + B_FALSE); + } + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + + if ((error == CRYPTO_SUCCESS) || (error == CRYPTO_QUEUED)) + *ctxp = (crypto_context_t)ctx; + else { + /* Release the hold done in kcf_new_ctx(). */ + KCF_CONTEXT_REFRELE((kcf_context_t *)ctx->cc_framework_private); + } + + return (error); +} + +/* + * Same as crypto_digest_init_prov(), but relies on the KCF scheduler + * to choose a provider. See crypto_digest_init_prov() comments for + * more information. + */ +int +crypto_digest_init(crypto_mechanism_t *mech, crypto_context_t *ctxp, + crypto_call_req_t *crq) +{ + int error; + kcf_provider_desc_t *pd; + kcf_prov_tried_t *list = NULL; + +retry: + /* The pd is returned held */ + if ((pd = kcf_get_mech_provider(mech->cm_type, NULL, &error, + list, CRYPTO_FG_DIGEST, CHECK_RESTRICT(crq), 0)) == NULL) { + if (list != NULL) + kcf_free_triedlist(list); + return (error); + } + + if (pd->pd_prov_type == CRYPTO_HW_PROVIDER && + (pd->pd_flags & CRYPTO_HASH_NO_UPDATE)) { + /* + * The hardware provider has limited digest support. + * So, we fallback early here to using a software provider. + * + * XXX - need to enhance to do the fallback later in + * crypto_digest_update() if the size of accumulated input data + * exceeds the maximum size digestable by hardware provider. + */ + error = CRYPTO_BUFFER_TOO_BIG; + } else { + error = crypto_digest_init_prov(pd, pd->pd_sid, + mech, ctxp, crq); + } + + if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED && + IS_RECOVERABLE(error)) { + /* Add pd to the linked list of providers tried. */ + if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL) + goto retry; + } + + if (list != NULL) + kcf_free_triedlist(list); + KCF_PROV_REFRELE(pd); + return (error); +} + +/* + * crypto_digest_update() + * + * Arguments: + * context: A crypto_context_t initialized by digest_init(). + * data: The part of message to be digested. + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs a + * part of a message digest operation. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_digest_update(crypto_context_t context, crypto_data_t *data, + crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_DIGEST_UPDATE(pd, ctx, data, NULL); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_UPDATE, + ctx->cc_session, NULL, NULL, data, NULL); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + return (error); +} + +/* + * crypto_digest_final() + * + * Arguments: + * context: A crypto_context_t initialized by digest_init(). + * digest: The storage for the digest. + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs the + * final part of a message digest operation. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_digest_final(crypto_context_t context, crypto_data_t *digest, + crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_DIGEST_FINAL(pd, ctx, digest, NULL); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_FINAL, + ctx->cc_session, NULL, NULL, NULL, digest); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + /* Release the hold done in kcf_new_ctx() during init step. */ + KCF_CONTEXT_COND_RELEASE(error, kcf_ctx); + return (error); +} + +/* + * Performs a digest update on the specified key. Note that there is + * no k-API crypto_digest_key() equivalent of this function. + */ +int +crypto_digest_key_prov(crypto_context_t context, crypto_key_t *key, + crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_DIGEST_KEY(pd, ctx, key, NULL); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_DIGEST_KEY, + ctx->cc_session, NULL, key, NULL, NULL); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + return (error); +} + +/* + * See comments for crypto_digest_update() and crypto_digest_final(). + */ +int +crypto_digest_single(crypto_context_t context, crypto_data_t *data, + crypto_data_t *digest, crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_DIGEST(pd, ctx, data, digest, NULL); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_DIGEST_OPS_PARAMS(¶ms, KCF_OP_SINGLE, pd->pd_sid, + NULL, NULL, data, digest); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + /* Release the hold done in kcf_new_ctx() during init step. */ + KCF_CONTEXT_COND_RELEASE(error, kcf_ctx); + return (error); +} + +#if defined(_KERNEL) && defined(HAVE_SPL) +EXPORT_SYMBOL(crypto_digest_prov); +EXPORT_SYMBOL(crypto_digest); +EXPORT_SYMBOL(crypto_digest_init_prov); +EXPORT_SYMBOL(crypto_digest_init); +EXPORT_SYMBOL(crypto_digest_update); +EXPORT_SYMBOL(crypto_digest_final); +EXPORT_SYMBOL(crypto_digest_key_prov); +EXPORT_SYMBOL(crypto_digest_single); +#endif diff --git a/module/icp/api/kcf_mac.c b/module/icp/api/kcf_mac.c new file mode 100644 index 000000000000..2b4691c0371e --- /dev/null +++ b/module/icp/api/kcf_mac.c @@ -0,0 +1,648 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include +#include + +#define CRYPTO_OPS_OFFSET(f) offsetof(crypto_ops_t, co_##f) +#define CRYPTO_MAC_OFFSET(f) offsetof(crypto_mac_ops_t, f) + +/* + * Message authentication codes routines. + */ + +/* + * The following are the possible returned values common to all the routines + * below. The applicability of some of these return values depends on the + * presence of the arguments. + * + * CRYPTO_SUCCESS: The operation completed successfully. + * CRYPTO_QUEUED: A request was submitted successfully. The callback + * routine will be called when the operation is done. + * CRYPTO_INVALID_MECH_NUMBER, CRYPTO_INVALID_MECH_PARAM, or + * CRYPTO_INVALID_MECH for problems with the 'mech'. + * CRYPTO_INVALID_DATA for bogus 'data' + * CRYPTO_HOST_MEMORY for failure to allocate memory to handle this work. + * CRYPTO_INVALID_CONTEXT: Not a valid context. + * CRYPTO_BUSY: Cannot process the request now. Schedule a + * crypto_bufcall(), or try later. + * CRYPTO_NOT_SUPPORTED and CRYPTO_MECH_NOT_SUPPORTED: No provider is + * capable of a function or a mechanism. + * CRYPTO_INVALID_KEY: bogus 'key' argument. + * CRYPTO_INVALID_MAC: bogus 'mac' argument. + */ + +/* + * crypto_mac_prov() + * + * Arguments: + * mech: crypto_mechanism_t pointer. + * mech_type is a valid value previously returned by + * crypto_mech2id(); + * When the mech's parameter is not NULL, its definition depends + * on the standard definition of the mechanism. + * key: pointer to a crypto_key_t structure. + * data: The message to compute the MAC for. + * mac: Storage for the MAC. The length needed depends on the mechanism. + * tmpl: a crypto_ctx_template_t, opaque template of a context of a + * MAC with the 'mech' using 'key'. 'tmpl' is created by + * a previous call to crypto_create_ctx_template(). + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs a + * single-part message authentication of 'data' with the mechanism + * 'mech', using * the key 'key', on the specified provider with + * the specified session id. + * When complete and successful, 'mac' will contain the message + * authentication code. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'crq'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_mac_prov(crypto_provider_t provider, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_data_t *data, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_data_t *mac, crypto_call_req_t *crq) +{ + kcf_req_params_t params; + kcf_provider_desc_t *pd = provider; + kcf_provider_desc_t *real_provider = pd; + int rv; + + ASSERT(KCF_PROV_REFHELD(pd)); + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + rv = kcf_get_hardware_provider(mech->cm_type, + CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd, + &real_provider, CRYPTO_FG_MAC_ATOMIC); + + if (rv != CRYPTO_SUCCESS) + return (rv); + } + + KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, sid, mech, key, + data, mac, tmpl); + rv = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE); + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + + return (rv); +} + +/* + * Same as crypto_mac_prov(), but relies on the KCF scheduler to choose + * a provider. See crypto_mac() comments for more information. + */ +int +crypto_mac(crypto_mechanism_t *mech, crypto_data_t *data, + crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac, + crypto_call_req_t *crq) +{ + int error; + kcf_mech_entry_t *me; + kcf_req_params_t params; + kcf_provider_desc_t *pd; + kcf_ctx_template_t *ctx_tmpl; + crypto_spi_ctx_template_t spi_ctx_tmpl = NULL; + kcf_prov_tried_t *list = NULL; + +retry: + /* The pd is returned held */ + if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error, + list, CRYPTO_FG_MAC_ATOMIC, CHECK_RESTRICT(crq), + data->cd_length)) == NULL) { + if (list != NULL) + kcf_free_triedlist(list); + return (error); + } + + /* + * For SW providers, check the validity of the context template + * It is very rare that the generation number mis-matches, so + * is acceptable to fail here, and let the consumer recover by + * freeing this tmpl and create a new one for the key and new SW + * provider + */ + if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && + ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) { + if (ctx_tmpl->ct_generation != me->me_gen_swprov) { + if (list != NULL) + kcf_free_triedlist(list); + KCF_PROV_REFRELE(pd); + return (CRYPTO_OLD_CTX_TEMPLATE); + } else { + spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl; + } + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(crq, pd)) { + crypto_mechanism_t lmech; + + lmech = *mech; + KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech); + + error = KCF_PROV_MAC_ATOMIC(pd, pd->pd_sid, &lmech, key, data, + mac, spi_ctx_tmpl, KCF_SWFP_RHNDL(crq)); + KCF_PROV_INCRSTATS(pd, error); + } else { + if (pd->pd_prov_type == CRYPTO_HW_PROVIDER && + (pd->pd_flags & CRYPTO_HASH_NO_UPDATE) && + (data->cd_length > pd->pd_hash_limit)) { + /* + * XXX - We need a check to see if this is indeed + * a HMAC. So far, all kernel clients use + * this interface only for HMAC. So, this is fine + * for now. + */ + error = CRYPTO_BUFFER_TOO_BIG; + } else { + KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_ATOMIC, + pd->pd_sid, mech, key, data, mac, spi_ctx_tmpl); + + error = kcf_submit_request(pd, NULL, crq, ¶ms, + KCF_ISDUALREQ(crq)); + } + } + + if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED && + IS_RECOVERABLE(error)) { + /* Add pd to the linked list of providers tried. */ + if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL) + goto retry; + } + + if (list != NULL) + kcf_free_triedlist(list); + + KCF_PROV_REFRELE(pd); + return (error); +} + +/* + * Single part operation to compute the MAC corresponding to the specified + * 'data' and to verify that it matches the MAC specified by 'mac'. + * The other arguments are the same as the function crypto_mac_prov(). + */ +int +crypto_mac_verify_prov(crypto_provider_t provider, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_data_t *data, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_data_t *mac, crypto_call_req_t *crq) +{ + kcf_req_params_t params; + kcf_provider_desc_t *pd = provider; + kcf_provider_desc_t *real_provider = pd; + int rv; + + ASSERT(KCF_PROV_REFHELD(pd)); + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + rv = kcf_get_hardware_provider(mech->cm_type, + CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd, + &real_provider, CRYPTO_FG_MAC_ATOMIC); + + if (rv != CRYPTO_SUCCESS) + return (rv); + } + + KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_MAC_VERIFY_ATOMIC, sid, mech, + key, data, mac, tmpl); + rv = kcf_submit_request(real_provider, NULL, crq, ¶ms, B_FALSE); + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + + return (rv); +} + +/* + * Same as crypto_mac_verify_prov(), but relies on the KCF scheduler to choose + * a provider. See crypto_mac_verify_prov() comments for more information. + */ +int +crypto_mac_verify(crypto_mechanism_t *mech, crypto_data_t *data, + crypto_key_t *key, crypto_ctx_template_t tmpl, crypto_data_t *mac, + crypto_call_req_t *crq) +{ + int error; + kcf_mech_entry_t *me; + kcf_req_params_t params; + kcf_provider_desc_t *pd; + kcf_ctx_template_t *ctx_tmpl; + crypto_spi_ctx_template_t spi_ctx_tmpl = NULL; + kcf_prov_tried_t *list = NULL; + +retry: + /* The pd is returned held */ + if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error, + list, CRYPTO_FG_MAC_ATOMIC, CHECK_RESTRICT(crq), + data->cd_length)) == NULL) { + if (list != NULL) + kcf_free_triedlist(list); + return (error); + } + + /* + * For SW providers, check the validity of the context template + * It is very rare that the generation number mis-matches, so + * is acceptable to fail here, and let the consumer recover by + * freeing this tmpl and create a new one for the key and new SW + * provider + */ + if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && + ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) { + if (ctx_tmpl->ct_generation != me->me_gen_swprov) { + if (list != NULL) + kcf_free_triedlist(list); + KCF_PROV_REFRELE(pd); + return (CRYPTO_OLD_CTX_TEMPLATE); + } else { + spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl; + } + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(crq, pd)) { + crypto_mechanism_t lmech; + + lmech = *mech; + KCF_SET_PROVIDER_MECHNUM(mech->cm_type, pd, &lmech); + + error = KCF_PROV_MAC_VERIFY_ATOMIC(pd, pd->pd_sid, &lmech, key, + data, mac, spi_ctx_tmpl, KCF_SWFP_RHNDL(crq)); + KCF_PROV_INCRSTATS(pd, error); + } else { + if (pd->pd_prov_type == CRYPTO_HW_PROVIDER && + (pd->pd_flags & CRYPTO_HASH_NO_UPDATE) && + (data->cd_length > pd->pd_hash_limit)) { + /* see comments in crypto_mac() */ + error = CRYPTO_BUFFER_TOO_BIG; + } else { + KCF_WRAP_MAC_OPS_PARAMS(¶ms, + KCF_OP_MAC_VERIFY_ATOMIC, pd->pd_sid, mech, + key, data, mac, spi_ctx_tmpl); + + error = kcf_submit_request(pd, NULL, crq, ¶ms, + KCF_ISDUALREQ(crq)); + } + } + + if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED && + IS_RECOVERABLE(error)) { + /* Add pd to the linked list of providers tried. */ + if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL) + goto retry; + } + + if (list != NULL) + kcf_free_triedlist(list); + + KCF_PROV_REFRELE(pd); + return (error); +} + +/* + * crypto_mac_init_prov() + * + * Arguments: + * pd: pointer to the descriptor of the provider to use for this + * operation. + * sid: provider session id. + * mech: crypto_mechanism_t pointer. + * mech_type is a valid value previously returned by + * crypto_mech2id(); + * When the mech's parameter is not NULL, its definition depends + * on the standard definition of the mechanism. + * key: pointer to a crypto_key_t structure. + * tmpl: a crypto_ctx_template_t, opaque template of a context of a + * MAC with the 'mech' using 'key'. 'tmpl' is created by + * a previous call to crypto_create_ctx_template(). + * ctxp: Pointer to a crypto_context_t. + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs the + * initialization of a MAC operation on the specified provider with + * the specified session. + * When possible and applicable, will internally use the pre-computed MAC + * context from the context template, tmpl. + * When complete and successful, 'ctxp' will contain a crypto_context_t + * valid for later calls to mac_update() and mac_final(). + * The caller should hold a reference on the specified provider + * descriptor before calling this function. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_mac_init_prov(crypto_provider_t provider, crypto_session_id_t sid, + crypto_mechanism_t *mech, crypto_key_t *key, crypto_spi_ctx_template_t tmpl, + crypto_context_t *ctxp, crypto_call_req_t *crq) +{ + int rv; + crypto_ctx_t *ctx; + kcf_req_params_t params; + kcf_provider_desc_t *pd = provider; + kcf_provider_desc_t *real_provider = pd; + + ASSERT(KCF_PROV_REFHELD(pd)); + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + rv = kcf_get_hardware_provider(mech->cm_type, + CRYPTO_MECH_INVALID, CHECK_RESTRICT(crq), pd, + &real_provider, CRYPTO_FG_MAC); + + if (rv != CRYPTO_SUCCESS) + return (rv); + } + + /* Allocate and initialize the canonical context */ + if ((ctx = kcf_new_ctx(crq, real_provider, sid)) == NULL) { + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + return (CRYPTO_HOST_MEMORY); + } + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(crq, pd)) { + crypto_mechanism_t lmech; + + lmech = *mech; + KCF_SET_PROVIDER_MECHNUM(mech->cm_type, real_provider, &lmech); + rv = KCF_PROV_MAC_INIT(real_provider, ctx, &lmech, key, tmpl, + KCF_SWFP_RHNDL(crq)); + KCF_PROV_INCRSTATS(pd, rv); + } else { + KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_INIT, sid, mech, key, + NULL, NULL, tmpl); + rv = kcf_submit_request(real_provider, ctx, crq, ¶ms, + B_FALSE); + } + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) + KCF_PROV_REFRELE(real_provider); + + if ((rv == CRYPTO_SUCCESS) || (rv == CRYPTO_QUEUED)) + *ctxp = (crypto_context_t)ctx; + else { + /* Release the hold done in kcf_new_ctx(). */ + KCF_CONTEXT_REFRELE((kcf_context_t *)ctx->cc_framework_private); + } + + return (rv); +} + +/* + * Same as crypto_mac_init_prov(), but relies on the KCF scheduler to + * choose a provider. See crypto_mac_init_prov() comments for more + * information. + */ +int +crypto_mac_init(crypto_mechanism_t *mech, crypto_key_t *key, + crypto_ctx_template_t tmpl, crypto_context_t *ctxp, + crypto_call_req_t *crq) +{ + int error; + kcf_mech_entry_t *me; + kcf_provider_desc_t *pd; + kcf_ctx_template_t *ctx_tmpl; + crypto_spi_ctx_template_t spi_ctx_tmpl = NULL; + kcf_prov_tried_t *list = NULL; + +retry: + /* The pd is returned held */ + if ((pd = kcf_get_mech_provider(mech->cm_type, &me, &error, + list, CRYPTO_FG_MAC, CHECK_RESTRICT(crq), 0)) == NULL) { + if (list != NULL) + kcf_free_triedlist(list); + return (error); + } + + /* + * For SW providers, check the validity of the context template + * It is very rare that the generation number mis-matches, so + * is acceptable to fail here, and let the consumer recover by + * freeing this tmpl and create a new one for the key and new SW + * provider + */ + + if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && + ((ctx_tmpl = (kcf_ctx_template_t *)tmpl) != NULL)) { + if (ctx_tmpl->ct_generation != me->me_gen_swprov) { + if (list != NULL) + kcf_free_triedlist(list); + KCF_PROV_REFRELE(pd); + return (CRYPTO_OLD_CTX_TEMPLATE); + } else { + spi_ctx_tmpl = ctx_tmpl->ct_prov_tmpl; + } + } + + if (pd->pd_prov_type == CRYPTO_HW_PROVIDER && + (pd->pd_flags & CRYPTO_HASH_NO_UPDATE)) { + /* + * The hardware provider has limited HMAC support. + * So, we fallback early here to using a software provider. + * + * XXX - need to enhance to do the fallback later in + * crypto_mac_update() if the size of accumulated input data + * exceeds the maximum size digestable by hardware provider. + */ + error = CRYPTO_BUFFER_TOO_BIG; + } else { + error = crypto_mac_init_prov(pd, pd->pd_sid, mech, key, + spi_ctx_tmpl, ctxp, crq); + } + if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED && + IS_RECOVERABLE(error)) { + /* Add pd to the linked list of providers tried. */ + if (kcf_insert_triedlist(&list, pd, KCF_KMFLAG(crq)) != NULL) + goto retry; + } + + if (list != NULL) + kcf_free_triedlist(list); + + KCF_PROV_REFRELE(pd); + return (error); +} + +/* + * crypto_mac_update() + * + * Arguments: + * context: A crypto_context_t initialized by mac_init(). + * data: The message part to be MAC'ed + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs a + * part of a MAC operation. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_mac_update(crypto_context_t context, crypto_data_t *data, + crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + kcf_req_params_t params; + int rv; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + rv = KCF_PROV_MAC_UPDATE(pd, ctx, data, NULL); + KCF_PROV_INCRSTATS(pd, rv); + } else { + KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_UPDATE, + ctx->cc_session, NULL, NULL, data, NULL, NULL); + rv = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + return (rv); +} + +/* + * crypto_mac_final() + * + * Arguments: + * context: A crypto_context_t initialized by mac_init(). + * mac: Storage for the message authentication code. + * cr: crypto_call_req_t calling conditions and call back info. + * + * Description: + * Asynchronously submits a request for, or synchronously performs a + * part of a message authentication operation. + * + * Context: + * Process or interrupt, according to the semantics dictated by the 'cr'. + * + * Returns: + * See comment in the beginning of the file. + */ +int +crypto_mac_final(crypto_context_t context, crypto_data_t *mac, + crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + kcf_req_params_t params; + int rv; + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + ASSERT(pd->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + rv = KCF_PROV_MAC_FINAL(pd, ctx, mac, NULL); + KCF_PROV_INCRSTATS(pd, rv); + } else { + KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_FINAL, + ctx->cc_session, NULL, NULL, NULL, mac, NULL); + rv = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + /* Release the hold done in kcf_new_ctx() during init step. */ + KCF_CONTEXT_COND_RELEASE(rv, kcf_ctx); + return (rv); +} + +/* + * See comments for crypto_mac_update() and crypto_mac_final(). + */ +int +crypto_mac_single(crypto_context_t context, crypto_data_t *data, + crypto_data_t *mac, crypto_call_req_t *cr) +{ + crypto_ctx_t *ctx = (crypto_ctx_t *)context; + kcf_context_t *kcf_ctx; + kcf_provider_desc_t *pd; + int error; + kcf_req_params_t params; + + + if ((ctx == NULL) || + ((kcf_ctx = (kcf_context_t *)ctx->cc_framework_private) == NULL) || + ((pd = kcf_ctx->kc_prov_desc) == NULL)) { + return (CRYPTO_INVALID_CONTEXT); + } + + + /* The fast path for SW providers. */ + if (CHECK_FASTPATH(cr, pd)) { + error = KCF_PROV_MAC(pd, ctx, data, mac, NULL); + KCF_PROV_INCRSTATS(pd, error); + } else { + KCF_WRAP_MAC_OPS_PARAMS(¶ms, KCF_OP_SINGLE, pd->pd_sid, + NULL, NULL, data, mac, NULL); + error = kcf_submit_request(pd, ctx, cr, ¶ms, B_FALSE); + } + + /* Release the hold done in kcf_new_ctx() during init step. */ + KCF_CONTEXT_COND_RELEASE(error, kcf_ctx); + return (error); +} + +#if defined(_KERNEL) && defined(HAVE_SPL) +EXPORT_SYMBOL(crypto_mac_prov); +EXPORT_SYMBOL(crypto_mac); +EXPORT_SYMBOL(crypto_mac_verify_prov); +EXPORT_SYMBOL(crypto_mac_verify); +EXPORT_SYMBOL(crypto_mac_init_prov); +EXPORT_SYMBOL(crypto_mac_init); +EXPORT_SYMBOL(crypto_mac_update); +EXPORT_SYMBOL(crypto_mac_final); +EXPORT_SYMBOL(crypto_mac_single); +#endif diff --git a/module/icp/api/kcf_miscapi.c b/module/icp/api/kcf_miscapi.c new file mode 100644 index 000000000000..09d50f7be176 --- /dev/null +++ b/module/icp/api/kcf_miscapi.c @@ -0,0 +1,127 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include + +/* + * All event subscribers are put on a list. kcf_notify_list_lock + * protects changes to this list. + * + * The following locking order is maintained in the code - The + * global kcf_notify_list_lock followed by the individual lock + * in a kcf_ntfy_elem structure (kn_lock). + */ +kmutex_t ntfy_list_lock; +kcondvar_t ntfy_list_cv; /* cv the service thread waits on */ +static kcf_ntfy_elem_t *ntfy_list_head; + +/* + * crypto_mech2id() + * + * Arguments: + * . mechname: A null-terminated string identifying the mechanism name. + * + * Description: + * Walks the mechanisms tables, looking for an entry that matches the + * mechname. Once it find it, it builds the 64-bit mech_type and returns + * it. If there are no hardware or software providers for the mechanism, + * but there is an unloaded software provider, this routine will attempt + * to load it. + * + * Context: + * Process and interruption. + * + * Returns: + * The unique mechanism identified by 'mechname', if found. + * CRYPTO_MECH_INVALID otherwise. + */ +crypto_mech_type_t +crypto_mech2id(char *mechname) +{ + return (crypto_mech2id_common(mechname, B_TRUE)); +} + +/* + * We walk the notification list and do the callbacks. + */ +void +kcf_walk_ntfylist(uint32_t event, void *event_arg) +{ + kcf_ntfy_elem_t *nep; + int nelem = 0; + + mutex_enter(&ntfy_list_lock); + + /* + * Count how many clients are on the notification list. We need + * this count to ensure that clients which joined the list after we + * have started this walk, are not wrongly notified. + */ + for (nep = ntfy_list_head; nep != NULL; nep = nep->kn_next) + nelem++; + + for (nep = ntfy_list_head; (nep != NULL && nelem); nep = nep->kn_next) { + nelem--; + + /* + * Check if this client is interested in the + * event. + */ + if (!(nep->kn_event_mask & event)) + continue; + + mutex_enter(&nep->kn_lock); + nep->kn_state = NTFY_RUNNING; + mutex_exit(&nep->kn_lock); + mutex_exit(&ntfy_list_lock); + + /* + * We invoke the callback routine with no locks held. Another + * client could have joined the list meanwhile. This is fine + * as we maintain nelem as stated above. The NULL check in the + * for loop guards against shrinkage. Also, any callers of + * crypto_unnotify_events() at this point cv_wait till kn_state + * changes to NTFY_WAITING. Hence, nep is assured to be valid. + */ + (*nep->kn_func)(event, event_arg); + + mutex_enter(&nep->kn_lock); + nep->kn_state = NTFY_WAITING; + cv_broadcast(&nep->kn_cv); + mutex_exit(&nep->kn_lock); + + mutex_enter(&ntfy_list_lock); + } + + mutex_exit(&ntfy_list_lock); +} + +#if defined(_KERNEL) && defined(HAVE_SPL) +EXPORT_SYMBOL(crypto_mech2id); +#endif diff --git a/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman b/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman new file mode 100644 index 000000000000..48fea7bb333e --- /dev/null +++ b/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman @@ -0,0 +1,23 @@ + --------------------------------------------------------------------------- + Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved. + + LICENSE TERMS + + The free distribution and use of this software is allowed (with or without + changes) provided that: + + 1. source code distributions include the above copyright notice, this + list of conditions and the following disclaimer; + + 2. binary distributions include the above copyright notice, this list + of conditions and the following disclaimer in their documentation; + + 3. the name of the copyright holder is not used to endorse products + built using this software without specific written permission. + + DISCLAIMER + + This software is provided 'as is' with no explicit or implied warranties + in respect of its properties, including, but not limited to, correctness + and/or fitness for purpose. + --------------------------------------------------------------------------- diff --git a/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman.descrip b/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman.descrip new file mode 100644 index 000000000000..5f822cf27586 --- /dev/null +++ b/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.gladman.descrip @@ -0,0 +1 @@ +PORTIONS OF AES FUNCTIONALITY diff --git a/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl b/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl new file mode 100644 index 000000000000..a2c4adcbe6a5 --- /dev/null +++ b/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl @@ -0,0 +1,127 @@ + + LICENSE ISSUES + ============== + + The OpenSSL toolkit stays under a dual license, i.e. both the conditions of + the OpenSSL License and the original SSLeay license apply to the toolkit. + See below for the actual license texts. Actually both licenses are BSD-style + Open Source licenses. In case of any license issues related to OpenSSL + please contact openssl-core@openssl.org. + + OpenSSL License + --------------- + +/* ==================================================================== + * Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + + Original SSLeay License + ----------------------- + +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ + diff --git a/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl.descrip b/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl.descrip new file mode 100644 index 000000000000..5f822cf27586 --- /dev/null +++ b/module/icp/asm-x86_64/aes/THIRDPARTYLICENSE.openssl.descrip @@ -0,0 +1 @@ +PORTIONS OF AES FUNCTIONALITY diff --git a/module/icp/asm-x86_64/aes/aes_amd64.S b/module/icp/asm-x86_64/aes/aes_amd64.S new file mode 100644 index 000000000000..fb64441196ca --- /dev/null +++ b/module/icp/asm-x86_64/aes/aes_amd64.S @@ -0,0 +1,900 @@ +/* + * --------------------------------------------------------------------------- + * Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved. + * + * LICENSE TERMS + * + * The free distribution and use of this software is allowed (with or without + * changes) provided that: + * + * 1. source code distributions include the above copyright notice, this + * list of conditions and the following disclaimer; + * + * 2. binary distributions include the above copyright notice, this list + * of conditions and the following disclaimer in their documentation; + * + * 3. the name of the copyright holder is not used to endorse products + * built using this software without specific written permission. + * + * DISCLAIMER + * + * This software is provided 'as is' with no explicit or implied warranties + * in respect of its properties, including, but not limited to, correctness + * and/or fitness for purpose. + * --------------------------------------------------------------------------- + * Issue 20/12/2007 + * + * I am grateful to Dag Arne Osvik for many discussions of the techniques that + * can be used to optimise AES assembler code on AMD64/EM64T architectures. + * Some of the techniques used in this implementation are the result of + * suggestions made by him for which I am most grateful. + * + * An AES implementation for AMD64 processors using the YASM assembler. This + * implementation provides only encryption, decryption and hence requires key + * scheduling support in C. It uses 8k bytes of tables but its encryption and + * decryption performance is very close to that obtained using large tables. + * It can use either MS Windows or Gnu/Linux/OpenSolaris OS calling conventions, + * which are as follows: + * ms windows gnu/linux/opensolaris os + * + * in_blk rcx rdi + * out_blk rdx rsi + * context (cx) r8 rdx + * + * preserved rsi - + rbx, rbp, rsp, r12, r13, r14 & r15 + * registers rdi - on both + * + * destroyed - rsi + rax, rcx, rdx, r8, r9, r10 & r11 + * registers - rdi on both + * + * The convention used here is that for gnu/linux/opensolaris os. + * + * This code provides the standard AES block size (128 bits, 16 bytes) and the + * three standard AES key sizes (128, 192 and 256 bits). It has the same call + * interface as my C implementation. It uses the Microsoft C AMD64 calling + * conventions in which the three parameters are placed in rcx, rdx and r8 + * respectively. The rbx, rsi, rdi, rbp and r12..r15 registers are preserved. + * + * OpenSolaris Note: + * Modified to use GNU/Linux/Solaris calling conventions. + * That is parameters are placed in rdi, rsi, rdx, and rcx, respectively. + * + * AES_RETURN aes_encrypt(const unsigned char in_blk[], + * unsigned char out_blk[], const aes_encrypt_ctx cx[1])/ + * + * AES_RETURN aes_decrypt(const unsigned char in_blk[], + * unsigned char out_blk[], const aes_decrypt_ctx cx[1])/ + * + * AES_RETURN aes_encrypt_key(const unsigned char key[], + * const aes_encrypt_ctx cx[1])/ + * + * AES_RETURN aes_decrypt_key(const unsigned char key[], + * const aes_decrypt_ctx cx[1])/ + * + * AES_RETURN aes_encrypt_key(const unsigned char key[], + * unsigned int len, const aes_decrypt_ctx cx[1])/ + * + * AES_RETURN aes_decrypt_key(const unsigned char key[], + * unsigned int len, const aes_decrypt_ctx cx[1])/ + * + * where is 128, 102 or 256. In the last two calls the length can be in + * either bits or bytes. + * + * Comment in/out the following lines to obtain the desired subroutines. These + * selections MUST match those in the C header file aesopt.h + */ +#define AES_REV_DKS /* define if key decryption schedule is reversed */ + +#define LAST_ROUND_TABLES /* define for the faster version using extra tables */ + +/* + * The encryption key schedule has the following in memory layout where N is the + * number of rounds (10, 12 or 14): + * + * lo: | input key (round 0) | / each round is four 32-bit words + * | encryption round 1 | + * | encryption round 2 | + * .... + * | encryption round N-1 | + * hi: | encryption round N | + * + * The decryption key schedule is normally set up so that it has the same + * layout as above by actually reversing the order of the encryption key + * schedule in memory (this happens when AES_REV_DKS is set): + * + * lo: | decryption round 0 | = | encryption round N | + * | decryption round 1 | = INV_MIX_COL[ | encryption round N-1 | ] + * | decryption round 2 | = INV_MIX_COL[ | encryption round N-2 | ] + * .... .... + * | decryption round N-1 | = INV_MIX_COL[ | encryption round 1 | ] + * hi: | decryption round N | = | input key (round 0) | + * + * with rounds except the first and last modified using inv_mix_column() + * But if AES_REV_DKS is NOT set the order of keys is left as it is for + * encryption so that it has to be accessed in reverse when used for + * decryption (although the inverse mix column modifications are done) + * + * lo: | decryption round 0 | = | input key (round 0) | + * | decryption round 1 | = INV_MIX_COL[ | encryption round 1 | ] + * | decryption round 2 | = INV_MIX_COL[ | encryption round 2 | ] + * .... .... + * | decryption round N-1 | = INV_MIX_COL[ | encryption round N-1 | ] + * hi: | decryption round N | = | encryption round N | + * + * This layout is faster when the assembler key scheduling provided here + * is used. + * + * End of user defines + */ + +/* + * --------------------------------------------------------------------------- + * OpenSolaris OS modifications + * + * This source originates from Brian Gladman file aes_amd64.asm + * in http://fp.gladman.plus.com/AES/aes-src-04-03-08.zip + * with these changes: + * + * 1. Removed MS Windows-specific code within DLL_EXPORT, _SEH_, and + * !__GNUC__ ifdefs. Also removed ENCRYPTION, DECRYPTION, + * AES_128, AES_192, AES_256, AES_VAR ifdefs. + * + * 2. Translate yasm/nasm %define and .macro definitions to cpp(1) #define + * + * 3. Translate yasm/nasm %ifdef/%ifndef to cpp(1) #ifdef + * + * 4. Translate Intel/yasm/nasm syntax to ATT/OpenSolaris as(1) syntax + * (operands reversed, literals prefixed with "$", registers prefixed with "%", + * and "[register+offset]", addressing changed to "offset(register)", + * parenthesis in constant expressions "()" changed to square brackets "[]", + * "." removed from local (numeric) labels, and other changes. + * Examples: + * Intel/yasm/nasm Syntax ATT/OpenSolaris Syntax + * mov rax,(4*20h) mov $[4*0x20],%rax + * mov rax,[ebx+20h] mov 0x20(%ebx),%rax + * lea rax,[ebx+ecx] lea (%ebx,%ecx),%rax + * sub rax,[ebx+ecx*4-20h] sub -0x20(%ebx,%ecx,4),%rax + * + * 5. Added OpenSolaris ENTRY_NP/SET_SIZE macros from + * /usr/include/sys/asm_linkage.h, lint(1B) guards, and dummy C function + * definitions for lint. + * + * 6. Renamed functions and reordered parameters to match OpenSolaris: + * Original Gladman interface: + * int aes_encrypt(const unsigned char *in, + * unsigned char *out, const aes_encrypt_ctx cx[1])/ + * int aes_decrypt(const unsigned char *in, + * unsigned char *out, const aes_encrypt_ctx cx[1])/ + * Note: aes_encrypt_ctx contains ks, a 60 element array of uint32_t, + * and a union type, inf., containing inf.l, a uint32_t and + * inf.b, a 4-element array of uint32_t. Only b[0] in the array (aka "l") is + * used and contains the key schedule length * 16 where key schedule length is + * 10, 12, or 14 bytes. + * + * OpenSolaris OS interface: + * void aes_encrypt_amd64(const aes_ks_t *ks, int Nr, + * const uint32_t pt[4], uint32_t ct[4])/ + * void aes_decrypt_amd64(const aes_ks_t *ks, int Nr, + * const uint32_t pt[4], uint32_t ct[4])/ + * typedef union {uint64_t ks64[(MAX_AES_NR + 1) * 4]/ + * uint32_t ks32[(MAX_AES_NR + 1) * 4]/ } aes_ks_t/ + * Note: ks is the AES key schedule, Nr is number of rounds, pt is plain text, + * ct is crypto text, and MAX_AES_NR is 14. + * For the x86 64-bit architecture, OpenSolaris OS uses ks32 instead of ks64. + */ + +#if defined(lint) || defined(__lint) + +#include +/* ARGSUSED */ +void +aes_encrypt_amd64(const uint32_t rk[], int Nr, const uint32_t pt[4], + uint32_t ct[4]) { +} +/* ARGSUSED */ +void +aes_decrypt_amd64(const uint32_t rk[], int Nr, const uint32_t ct[4], + uint32_t pt[4]) { +} + + +#else + +#define _ASM +#include + +#define KS_LENGTH 60 + +#define raxd eax +#define rdxd edx +#define rcxd ecx +#define rbxd ebx +#define rsid esi +#define rdid edi + +#define raxb al +#define rdxb dl +#define rcxb cl +#define rbxb bl +#define rsib sil +#define rdib dil + +// finite field multiplies by {02}, {04} and {08} + +#define f2(x) [[x<<1]^[[[x>>7]&1]*0x11b]] +#define f4(x) [[x<<2]^[[[x>>6]&1]*0x11b]^[[[x>>6]&2]*0x11b]] +#define f8(x) [[x<<3]^[[[x>>5]&1]*0x11b]^[[[x>>5]&2]*0x11b]^[[[x>>5]&4]*0x11b]] + +// finite field multiplies required in table generation + +#define f3(x) [[f2(x)] ^ [x]] +#define f9(x) [[f8(x)] ^ [x]] +#define fb(x) [[f8(x)] ^ [f2(x)] ^ [x]] +#define fd(x) [[f8(x)] ^ [f4(x)] ^ [x]] +#define fe(x) [[f8(x)] ^ [f4(x)] ^ [f2(x)]] + +// macros for expanding S-box data + +#define u8(x) [f2(x)], [x], [x], [f3(x)], [f2(x)], [x], [x], [f3(x)] +#define v8(x) [fe(x)], [f9(x)], [fd(x)], [fb(x)], [fe(x)], [f9(x)], [fd(x)], [x] +#define w8(x) [x], 0, 0, 0, [x], 0, 0, 0 + +#define enc_vals(x) \ + .byte x(0x63),x(0x7c),x(0x77),x(0x7b),x(0xf2),x(0x6b),x(0x6f),x(0xc5); \ + .byte x(0x30),x(0x01),x(0x67),x(0x2b),x(0xfe),x(0xd7),x(0xab),x(0x76); \ + .byte x(0xca),x(0x82),x(0xc9),x(0x7d),x(0xfa),x(0x59),x(0x47),x(0xf0); \ + .byte x(0xad),x(0xd4),x(0xa2),x(0xaf),x(0x9c),x(0xa4),x(0x72),x(0xc0); \ + .byte x(0xb7),x(0xfd),x(0x93),x(0x26),x(0x36),x(0x3f),x(0xf7),x(0xcc); \ + .byte x(0x34),x(0xa5),x(0xe5),x(0xf1),x(0x71),x(0xd8),x(0x31),x(0x15); \ + .byte x(0x04),x(0xc7),x(0x23),x(0xc3),x(0x18),x(0x96),x(0x05),x(0x9a); \ + .byte x(0x07),x(0x12),x(0x80),x(0xe2),x(0xeb),x(0x27),x(0xb2),x(0x75); \ + .byte x(0x09),x(0x83),x(0x2c),x(0x1a),x(0x1b),x(0x6e),x(0x5a),x(0xa0); \ + .byte x(0x52),x(0x3b),x(0xd6),x(0xb3),x(0x29),x(0xe3),x(0x2f),x(0x84); \ + .byte x(0x53),x(0xd1),x(0x00),x(0xed),x(0x20),x(0xfc),x(0xb1),x(0x5b); \ + .byte x(0x6a),x(0xcb),x(0xbe),x(0x39),x(0x4a),x(0x4c),x(0x58),x(0xcf); \ + .byte x(0xd0),x(0xef),x(0xaa),x(0xfb),x(0x43),x(0x4d),x(0x33),x(0x85); \ + .byte x(0x45),x(0xf9),x(0x02),x(0x7f),x(0x50),x(0x3c),x(0x9f),x(0xa8); \ + .byte x(0x51),x(0xa3),x(0x40),x(0x8f),x(0x92),x(0x9d),x(0x38),x(0xf5); \ + .byte x(0xbc),x(0xb6),x(0xda),x(0x21),x(0x10),x(0xff),x(0xf3),x(0xd2); \ + .byte x(0xcd),x(0x0c),x(0x13),x(0xec),x(0x5f),x(0x97),x(0x44),x(0x17); \ + .byte x(0xc4),x(0xa7),x(0x7e),x(0x3d),x(0x64),x(0x5d),x(0x19),x(0x73); \ + .byte x(0x60),x(0x81),x(0x4f),x(0xdc),x(0x22),x(0x2a),x(0x90),x(0x88); \ + .byte x(0x46),x(0xee),x(0xb8),x(0x14),x(0xde),x(0x5e),x(0x0b),x(0xdb); \ + .byte x(0xe0),x(0x32),x(0x3a),x(0x0a),x(0x49),x(0x06),x(0x24),x(0x5c); \ + .byte x(0xc2),x(0xd3),x(0xac),x(0x62),x(0x91),x(0x95),x(0xe4),x(0x79); \ + .byte x(0xe7),x(0xc8),x(0x37),x(0x6d),x(0x8d),x(0xd5),x(0x4e),x(0xa9); \ + .byte x(0x6c),x(0x56),x(0xf4),x(0xea),x(0x65),x(0x7a),x(0xae),x(0x08); \ + .byte x(0xba),x(0x78),x(0x25),x(0x2e),x(0x1c),x(0xa6),x(0xb4),x(0xc6); \ + .byte x(0xe8),x(0xdd),x(0x74),x(0x1f),x(0x4b),x(0xbd),x(0x8b),x(0x8a); \ + .byte x(0x70),x(0x3e),x(0xb5),x(0x66),x(0x48),x(0x03),x(0xf6),x(0x0e); \ + .byte x(0x61),x(0x35),x(0x57),x(0xb9),x(0x86),x(0xc1),x(0x1d),x(0x9e); \ + .byte x(0xe1),x(0xf8),x(0x98),x(0x11),x(0x69),x(0xd9),x(0x8e),x(0x94); \ + .byte x(0x9b),x(0x1e),x(0x87),x(0xe9),x(0xce),x(0x55),x(0x28),x(0xdf); \ + .byte x(0x8c),x(0xa1),x(0x89),x(0x0d),x(0xbf),x(0xe6),x(0x42),x(0x68); \ + .byte x(0x41),x(0x99),x(0x2d),x(0x0f),x(0xb0),x(0x54),x(0xbb),x(0x16) + +#define dec_vals(x) \ + .byte x(0x52),x(0x09),x(0x6a),x(0xd5),x(0x30),x(0x36),x(0xa5),x(0x38); \ + .byte x(0xbf),x(0x40),x(0xa3),x(0x9e),x(0x81),x(0xf3),x(0xd7),x(0xfb); \ + .byte x(0x7c),x(0xe3),x(0x39),x(0x82),x(0x9b),x(0x2f),x(0xff),x(0x87); \ + .byte x(0x34),x(0x8e),x(0x43),x(0x44),x(0xc4),x(0xde),x(0xe9),x(0xcb); \ + .byte x(0x54),x(0x7b),x(0x94),x(0x32),x(0xa6),x(0xc2),x(0x23),x(0x3d); \ + .byte x(0xee),x(0x4c),x(0x95),x(0x0b),x(0x42),x(0xfa),x(0xc3),x(0x4e); \ + .byte x(0x08),x(0x2e),x(0xa1),x(0x66),x(0x28),x(0xd9),x(0x24),x(0xb2); \ + .byte x(0x76),x(0x5b),x(0xa2),x(0x49),x(0x6d),x(0x8b),x(0xd1),x(0x25); \ + .byte x(0x72),x(0xf8),x(0xf6),x(0x64),x(0x86),x(0x68),x(0x98),x(0x16); \ + .byte x(0xd4),x(0xa4),x(0x5c),x(0xcc),x(0x5d),x(0x65),x(0xb6),x(0x92); \ + .byte x(0x6c),x(0x70),x(0x48),x(0x50),x(0xfd),x(0xed),x(0xb9),x(0xda); \ + .byte x(0x5e),x(0x15),x(0x46),x(0x57),x(0xa7),x(0x8d),x(0x9d),x(0x84); \ + .byte x(0x90),x(0xd8),x(0xab),x(0x00),x(0x8c),x(0xbc),x(0xd3),x(0x0a); \ + .byte x(0xf7),x(0xe4),x(0x58),x(0x05),x(0xb8),x(0xb3),x(0x45),x(0x06); \ + .byte x(0xd0),x(0x2c),x(0x1e),x(0x8f),x(0xca),x(0x3f),x(0x0f),x(0x02); \ + .byte x(0xc1),x(0xaf),x(0xbd),x(0x03),x(0x01),x(0x13),x(0x8a),x(0x6b); \ + .byte x(0x3a),x(0x91),x(0x11),x(0x41),x(0x4f),x(0x67),x(0xdc),x(0xea); \ + .byte x(0x97),x(0xf2),x(0xcf),x(0xce),x(0xf0),x(0xb4),x(0xe6),x(0x73); \ + .byte x(0x96),x(0xac),x(0x74),x(0x22),x(0xe7),x(0xad),x(0x35),x(0x85); \ + .byte x(0xe2),x(0xf9),x(0x37),x(0xe8),x(0x1c),x(0x75),x(0xdf),x(0x6e); \ + .byte x(0x47),x(0xf1),x(0x1a),x(0x71),x(0x1d),x(0x29),x(0xc5),x(0x89); \ + .byte x(0x6f),x(0xb7),x(0x62),x(0x0e),x(0xaa),x(0x18),x(0xbe),x(0x1b); \ + .byte x(0xfc),x(0x56),x(0x3e),x(0x4b),x(0xc6),x(0xd2),x(0x79),x(0x20); \ + .byte x(0x9a),x(0xdb),x(0xc0),x(0xfe),x(0x78),x(0xcd),x(0x5a),x(0xf4); \ + .byte x(0x1f),x(0xdd),x(0xa8),x(0x33),x(0x88),x(0x07),x(0xc7),x(0x31); \ + .byte x(0xb1),x(0x12),x(0x10),x(0x59),x(0x27),x(0x80),x(0xec),x(0x5f); \ + .byte x(0x60),x(0x51),x(0x7f),x(0xa9),x(0x19),x(0xb5),x(0x4a),x(0x0d); \ + .byte x(0x2d),x(0xe5),x(0x7a),x(0x9f),x(0x93),x(0xc9),x(0x9c),x(0xef); \ + .byte x(0xa0),x(0xe0),x(0x3b),x(0x4d),x(0xae),x(0x2a),x(0xf5),x(0xb0); \ + .byte x(0xc8),x(0xeb),x(0xbb),x(0x3c),x(0x83),x(0x53),x(0x99),x(0x61); \ + .byte x(0x17),x(0x2b),x(0x04),x(0x7e),x(0xba),x(0x77),x(0xd6),x(0x26); \ + .byte x(0xe1),x(0x69),x(0x14),x(0x63),x(0x55),x(0x21),x(0x0c),x(0x7d) + +#define tptr %rbp /* table pointer */ +#define kptr %r8 /* key schedule pointer */ +#define fofs 128 /* adjust offset in key schedule to keep |disp| < 128 */ +#define fk_ref(x, y) -16*x+fofs+4*y(kptr) + +#ifdef AES_REV_DKS +#define rofs 128 +#define ik_ref(x, y) -16*x+rofs+4*y(kptr) + +#else +#define rofs -128 +#define ik_ref(x, y) 16*x+rofs+4*y(kptr) +#endif /* AES_REV_DKS */ + +#define tab_0(x) (tptr,x,8) +#define tab_1(x) 3(tptr,x,8) +#define tab_2(x) 2(tptr,x,8) +#define tab_3(x) 1(tptr,x,8) +#define tab_f(x) 1(tptr,x,8) +#define tab_i(x) 7(tptr,x,8) + +#define ff_rnd(p1, p2, p3, p4, round) /* normal forward round */ \ + mov fk_ref(round,0), p1; \ + mov fk_ref(round,1), p2; \ + mov fk_ref(round,2), p3; \ + mov fk_ref(round,3), p4; \ + \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + shr $16, %eax; \ + xor tab_0(%rsi), p1; \ + xor tab_1(%rdi), p4; \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + xor tab_2(%rsi), p3; \ + xor tab_3(%rdi), p2; \ + \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + shr $16, %ebx; \ + xor tab_0(%rsi), p2; \ + xor tab_1(%rdi), p1; \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + xor tab_2(%rsi), p4; \ + xor tab_3(%rdi), p3; \ + \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + shr $16, %ecx; \ + xor tab_0(%rsi), p3; \ + xor tab_1(%rdi), p2; \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + xor tab_2(%rsi), p1; \ + xor tab_3(%rdi), p4; \ + \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + shr $16, %edx; \ + xor tab_0(%rsi), p4; \ + xor tab_1(%rdi), p3; \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + xor tab_2(%rsi), p2; \ + xor tab_3(%rdi), p1; \ + \ + mov p1, %eax; \ + mov p2, %ebx; \ + mov p3, %ecx; \ + mov p4, %edx + +#ifdef LAST_ROUND_TABLES + +#define fl_rnd(p1, p2, p3, p4, round) /* last forward round */ \ + add $2048, tptr; \ + mov fk_ref(round,0), p1; \ + mov fk_ref(round,1), p2; \ + mov fk_ref(round,2), p3; \ + mov fk_ref(round,3), p4; \ + \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + shr $16, %eax; \ + xor tab_0(%rsi), p1; \ + xor tab_1(%rdi), p4; \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + xor tab_2(%rsi), p3; \ + xor tab_3(%rdi), p2; \ + \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + shr $16, %ebx; \ + xor tab_0(%rsi), p2; \ + xor tab_1(%rdi), p1; \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + xor tab_2(%rsi), p4; \ + xor tab_3(%rdi), p3; \ + \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + shr $16, %ecx; \ + xor tab_0(%rsi), p3; \ + xor tab_1(%rdi), p2; \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + xor tab_2(%rsi), p1; \ + xor tab_3(%rdi), p4; \ + \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + shr $16, %edx; \ + xor tab_0(%rsi), p4; \ + xor tab_1(%rdi), p3; \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + xor tab_2(%rsi), p2; \ + xor tab_3(%rdi), p1 + +#else + +#define fl_rnd(p1, p2, p3, p4, round) /* last forward round */ \ + mov fk_ref(round,0), p1; \ + mov fk_ref(round,1), p2; \ + mov fk_ref(round,2), p3; \ + mov fk_ref(round,3), p4; \ + \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + shr $16, %eax; \ + movzx tab_f(%rsi), %esi; \ + movzx tab_f(%rdi), %edi; \ + xor %esi, p1; \ + rol $8, %edi; \ + xor %edi, p4; \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + movzx tab_f(%rsi), %esi; \ + movzx tab_f(%rdi), %edi; \ + rol $16, %esi; \ + rol $24, %edi; \ + xor %esi, p3; \ + xor %edi, p2; \ + \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + shr $16, %ebx; \ + movzx tab_f(%rsi), %esi; \ + movzx tab_f(%rdi), %edi; \ + xor %esi, p2; \ + rol $8, %edi; \ + xor %edi, p1; \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + movzx tab_f(%rsi), %esi; \ + movzx tab_f(%rdi), %edi; \ + rol $16, %esi; \ + rol $24, %edi; \ + xor %esi, p4; \ + xor %edi, p3; \ + \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + movzx tab_f(%rsi), %esi; \ + movzx tab_f(%rdi), %edi; \ + shr $16, %ecx; \ + xor %esi, p3; \ + rol $8, %edi; \ + xor %edi, p2; \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + movzx tab_f(%rsi), %esi; \ + movzx tab_f(%rdi), %edi; \ + rol $16, %esi; \ + rol $24, %edi; \ + xor %esi, p1; \ + xor %edi, p4; \ + \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + movzx tab_f(%rsi), %esi; \ + movzx tab_f(%rdi), %edi; \ + shr $16, %edx; \ + xor %esi, p4; \ + rol $8, %edi; \ + xor %edi, p3; \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + movzx tab_f(%rsi), %esi; \ + movzx tab_f(%rdi), %edi; \ + rol $16, %esi; \ + rol $24, %edi; \ + xor %esi, p2; \ + xor %edi, p1 + +#endif /* LAST_ROUND_TABLES */ + +#define ii_rnd(p1, p2, p3, p4, round) /* normal inverse round */ \ + mov ik_ref(round,0), p1; \ + mov ik_ref(round,1), p2; \ + mov ik_ref(round,2), p3; \ + mov ik_ref(round,3), p4; \ + \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + shr $16, %eax; \ + xor tab_0(%rsi), p1; \ + xor tab_1(%rdi), p2; \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + xor tab_2(%rsi), p3; \ + xor tab_3(%rdi), p4; \ + \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + shr $16, %ebx; \ + xor tab_0(%rsi), p2; \ + xor tab_1(%rdi), p3; \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + xor tab_2(%rsi), p4; \ + xor tab_3(%rdi), p1; \ + \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + shr $16, %ecx; \ + xor tab_0(%rsi), p3; \ + xor tab_1(%rdi), p4; \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + xor tab_2(%rsi), p1; \ + xor tab_3(%rdi), p2; \ + \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + shr $16, %edx; \ + xor tab_0(%rsi), p4; \ + xor tab_1(%rdi), p1; \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + xor tab_2(%rsi), p2; \ + xor tab_3(%rdi), p3; \ + \ + mov p1, %eax; \ + mov p2, %ebx; \ + mov p3, %ecx; \ + mov p4, %edx + +#ifdef LAST_ROUND_TABLES + +#define il_rnd(p1, p2, p3, p4, round) /* last inverse round */ \ + add $2048, tptr; \ + mov ik_ref(round,0), p1; \ + mov ik_ref(round,1), p2; \ + mov ik_ref(round,2), p3; \ + mov ik_ref(round,3), p4; \ + \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + shr $16, %eax; \ + xor tab_0(%rsi), p1; \ + xor tab_1(%rdi), p2; \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + xor tab_2(%rsi), p3; \ + xor tab_3(%rdi), p4; \ + \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + shr $16, %ebx; \ + xor tab_0(%rsi), p2; \ + xor tab_1(%rdi), p3; \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + xor tab_2(%rsi), p4; \ + xor tab_3(%rdi), p1; \ + \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + shr $16, %ecx; \ + xor tab_0(%rsi), p3; \ + xor tab_1(%rdi), p4; \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + xor tab_2(%rsi), p1; \ + xor tab_3(%rdi), p2; \ + \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + shr $16, %edx; \ + xor tab_0(%rsi), p4; \ + xor tab_1(%rdi), p1; \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + xor tab_2(%rsi), p2; \ + xor tab_3(%rdi), p3 + +#else + +#define il_rnd(p1, p2, p3, p4, round) /* last inverse round */ \ + mov ik_ref(round,0), p1; \ + mov ik_ref(round,1), p2; \ + mov ik_ref(round,2), p3; \ + mov ik_ref(round,3), p4; \ + \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + movzx tab_i(%rsi), %esi; \ + movzx tab_i(%rdi), %edi; \ + shr $16, %eax; \ + xor %esi, p1; \ + rol $8, %edi; \ + xor %edi, p2; \ + movzx %al, %esi; \ + movzx %ah, %edi; \ + movzx tab_i(%rsi), %esi; \ + movzx tab_i(%rdi), %edi; \ + rol $16, %esi; \ + rol $24, %edi; \ + xor %esi, p3; \ + xor %edi, p4; \ + \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + movzx tab_i(%rsi), %esi; \ + movzx tab_i(%rdi), %edi; \ + shr $16, %ebx; \ + xor %esi, p2; \ + rol $8, %edi; \ + xor %edi, p3; \ + movzx %bl, %esi; \ + movzx %bh, %edi; \ + movzx tab_i(%rsi), %esi; \ + movzx tab_i(%rdi), %edi; \ + rol $16, %esi; \ + rol $24, %edi; \ + xor %esi, p4; \ + xor %edi, p1; \ + \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + movzx tab_i(%rsi), %esi; \ + movzx tab_i(%rdi), %edi; \ + shr $16, %ecx; \ + xor %esi, p3; \ + rol $8, %edi; \ + xor %edi, p4; \ + movzx %cl, %esi; \ + movzx %ch, %edi; \ + movzx tab_i(%rsi), %esi; \ + movzx tab_i(%rdi), %edi; \ + rol $16, %esi; \ + rol $24, %edi; \ + xor %esi, p1; \ + xor %edi, p2; \ + \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + movzx tab_i(%rsi), %esi; \ + movzx tab_i(%rdi), %edi; \ + shr $16, %edx; \ + xor %esi, p4; \ + rol $8, %edi; \ + xor %edi, p1; \ + movzx %dl, %esi; \ + movzx %dh, %edi; \ + movzx tab_i(%rsi), %esi; \ + movzx tab_i(%rdi), %edi; \ + rol $16, %esi; \ + rol $24, %edi; \ + xor %esi, p2; \ + xor %edi, p3 + +#endif /* LAST_ROUND_TABLES */ + +/* + * OpenSolaris OS: + * void aes_encrypt_amd64(const aes_ks_t *ks, int Nr, + * const uint32_t pt[4], uint32_t ct[4])/ + * + * Original interface: + * int aes_encrypt(const unsigned char *in, + * unsigned char *out, const aes_encrypt_ctx cx[1])/ + */ + .align 64 +enc_tab: + enc_vals(u8) +#ifdef LAST_ROUND_TABLES + // Last Round Tables: + enc_vals(w8) +#endif + + + ENTRY_NP(aes_encrypt_amd64) +#ifdef GLADMAN_INTERFACE + // Original interface + sub $[4*8], %rsp // gnu/linux/opensolaris binary interface + mov %rsi, (%rsp) // output pointer (P2) + mov %rdx, %r8 // context (P3) + + mov %rbx, 1*8(%rsp) // P1: input pointer in rdi + mov %rbp, 2*8(%rsp) // P2: output pointer in (rsp) + mov %r12, 3*8(%rsp) // P3: context in r8 + movzx 4*KS_LENGTH(kptr), %esi // Get byte key length * 16 + +#else + // OpenSolaris OS interface + sub $[4*8], %rsp // Make room on stack to save registers + mov %rcx, (%rsp) // Save output pointer (P4) on stack + mov %rdi, %r8 // context (P1) + mov %rdx, %rdi // P3: save input pointer + shl $4, %esi // P2: esi byte key length * 16 + + mov %rbx, 1*8(%rsp) // Save registers + mov %rbp, 2*8(%rsp) + mov %r12, 3*8(%rsp) + // P1: context in r8 + // P2: byte key length * 16 in esi + // P3: input pointer in rdi + // P4: output pointer in (rsp) +#endif /* GLADMAN_INTERFACE */ + + lea enc_tab(%rip), tptr + sub $fofs, kptr + + // Load input block into registers + mov (%rdi), %eax + mov 1*4(%rdi), %ebx + mov 2*4(%rdi), %ecx + mov 3*4(%rdi), %edx + + xor fofs(kptr), %eax + xor fofs+4(kptr), %ebx + xor fofs+8(kptr), %ecx + xor fofs+12(kptr), %edx + + lea (kptr,%rsi), kptr + // Jump based on byte key length * 16: + cmp $[10*16], %esi + je 3f + cmp $[12*16], %esi + je 2f + cmp $[14*16], %esi + je 1f + mov $-1, %rax // error + jmp 4f + + // Perform normal forward rounds +1: ff_rnd(%r9d, %r10d, %r11d, %r12d, 13) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 12) +2: ff_rnd(%r9d, %r10d, %r11d, %r12d, 11) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 10) +3: ff_rnd(%r9d, %r10d, %r11d, %r12d, 9) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 8) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 7) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 6) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 5) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 4) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 3) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 2) + ff_rnd(%r9d, %r10d, %r11d, %r12d, 1) + fl_rnd(%r9d, %r10d, %r11d, %r12d, 0) + + // Copy results + mov (%rsp), %rbx + mov %r9d, (%rbx) + mov %r10d, 4(%rbx) + mov %r11d, 8(%rbx) + mov %r12d, 12(%rbx) + xor %rax, %rax +4: // Restore registers + mov 1*8(%rsp), %rbx + mov 2*8(%rsp), %rbp + mov 3*8(%rsp), %r12 + add $[4*8], %rsp + ret + + SET_SIZE(aes_encrypt_amd64) + +/* + * OpenSolaris OS: + * void aes_decrypt_amd64(const aes_ks_t *ks, int Nr, + * const uint32_t pt[4], uint32_t ct[4])/ + * + * Original interface: + * int aes_decrypt(const unsigned char *in, + * unsigned char *out, const aes_encrypt_ctx cx[1])/ + */ + .align 64 +dec_tab: + dec_vals(v8) +#ifdef LAST_ROUND_TABLES + // Last Round Tables: + dec_vals(w8) +#endif + + + ENTRY_NP(aes_decrypt_amd64) +#ifdef GLADMAN_INTERFACE + // Original interface + sub $[4*8], %rsp // gnu/linux/opensolaris binary interface + mov %rsi, (%rsp) // output pointer (P2) + mov %rdx, %r8 // context (P3) + + mov %rbx, 1*8(%rsp) // P1: input pointer in rdi + mov %rbp, 2*8(%rsp) // P2: output pointer in (rsp) + mov %r12, 3*8(%rsp) // P3: context in r8 + movzx 4*KS_LENGTH(kptr), %esi // Get byte key length * 16 + +#else + // OpenSolaris OS interface + sub $[4*8], %rsp // Make room on stack to save registers + mov %rcx, (%rsp) // Save output pointer (P4) on stack + mov %rdi, %r8 // context (P1) + mov %rdx, %rdi // P3: save input pointer + shl $4, %esi // P2: esi byte key length * 16 + + mov %rbx, 1*8(%rsp) // Save registers + mov %rbp, 2*8(%rsp) + mov %r12, 3*8(%rsp) + // P1: context in r8 + // P2: byte key length * 16 in esi + // P3: input pointer in rdi + // P4: output pointer in (rsp) +#endif /* GLADMAN_INTERFACE */ + + lea dec_tab(%rip), tptr + sub $rofs, kptr + + // Load input block into registers + mov (%rdi), %eax + mov 1*4(%rdi), %ebx + mov 2*4(%rdi), %ecx + mov 3*4(%rdi), %edx + +#ifdef AES_REV_DKS + mov kptr, %rdi + lea (kptr,%rsi), kptr +#else + lea (kptr,%rsi), %rdi +#endif + + xor rofs(%rdi), %eax + xor rofs+4(%rdi), %ebx + xor rofs+8(%rdi), %ecx + xor rofs+12(%rdi), %edx + + // Jump based on byte key length * 16: + cmp $[10*16], %esi + je 3f + cmp $[12*16], %esi + je 2f + cmp $[14*16], %esi + je 1f + mov $-1, %rax // error + jmp 4f + + // Perform normal inverse rounds +1: ii_rnd(%r9d, %r10d, %r11d, %r12d, 13) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 12) +2: ii_rnd(%r9d, %r10d, %r11d, %r12d, 11) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 10) +3: ii_rnd(%r9d, %r10d, %r11d, %r12d, 9) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 8) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 7) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 6) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 5) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 4) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 3) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 2) + ii_rnd(%r9d, %r10d, %r11d, %r12d, 1) + il_rnd(%r9d, %r10d, %r11d, %r12d, 0) + + // Copy results + mov (%rsp), %rbx + mov %r9d, (%rbx) + mov %r10d, 4(%rbx) + mov %r11d, 8(%rbx) + mov %r12d, 12(%rbx) + xor %rax, %rax +4: // Restore registers + mov 1*8(%rsp), %rbx + mov 2*8(%rsp), %rbp + mov 3*8(%rsp), %r12 + add $[4*8], %rsp + ret + + SET_SIZE(aes_decrypt_amd64) +#endif /* lint || __lint */ diff --git a/module/icp/asm-x86_64/aes/aes_intel.S b/module/icp/asm-x86_64/aes/aes_intel.S new file mode 100644 index 000000000000..0b4700f963d4 --- /dev/null +++ b/module/icp/asm-x86_64/aes/aes_intel.S @@ -0,0 +1,851 @@ +/* + * ==================================================================== + * Written by Intel Corporation for the OpenSSL project to add support + * for Intel AES-NI instructions. Rights for redistribution and usage + * in source and binary forms are granted according to the OpenSSL + * license. + * + * Author: Huang Ying + * Vinodh Gopal + * Kahraman Akdemir + * + * Intel AES-NI is a new set of Single Instruction Multiple Data (SIMD) + * instructions that are going to be introduced in the next generation + * of Intel processor, as of 2009. These instructions enable fast and + * secure data encryption and decryption, using the Advanced Encryption + * Standard (AES), defined by FIPS Publication number 197. The + * architecture introduces six instructions that offer full hardware + * support for AES. Four of them support high performance data + * encryption and decryption, and the other two instructions support + * the AES key expansion procedure. + * ==================================================================== + */ + +/* + * ==================================================================== + * Copyright (c) 1998-2008 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + */ + +/* + * ==================================================================== + * OpenSolaris OS modifications + * + * This source originates as files aes-intel.S and eng_aesni_asm.pl, in + * patches sent sent Dec. 9, 2008 and Dec. 24, 2008, respectively, by + * Huang Ying of Intel to the openssl-dev mailing list under the subject + * of "Add support to Intel AES-NI instruction set for x86_64 platform". + * + * This OpenSolaris version has these major changes from the original source: + * + * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from + * /usr/include/sys/asm_linkage.h, lint(1B) guards, and dummy C function + * definitions for lint. + * + * 2. Formatted code, added comments, and added #includes and #defines. + * + * 3. If bit CR0.TS is set, clear and set the TS bit, after and before + * calling kpreempt_disable() and kpreempt_enable(). + * If the TS bit is not set, Save and restore %xmm registers at the beginning + * and end of function calls (%xmm* registers are not saved and restored by + * during kernel thread preemption). + * + * 4. Renamed functions, reordered parameters, and changed return value + * to match OpenSolaris: + * + * OpenSSL interface: + * int intel_AES_set_encrypt_key(const unsigned char *userKey, + * const int bits, AES_KEY *key); + * int intel_AES_set_decrypt_key(const unsigned char *userKey, + * const int bits, AES_KEY *key); + * Return values for above are non-zero on error, 0 on success. + * + * void intel_AES_encrypt(const unsigned char *in, unsigned char *out, + * const AES_KEY *key); + * void intel_AES_decrypt(const unsigned char *in, unsigned char *out, + * const AES_KEY *key); + * typedef struct aes_key_st { + * unsigned int rd_key[4 *(AES_MAXNR + 1)]; + * int rounds; + * unsigned int pad[3]; + * } AES_KEY; + * Note: AES_LONG is undefined (that is, Intel uses 32-bit key schedules + * (ks32) instead of 64-bit (ks64). + * Number of rounds (aka round count) is at offset 240 of AES_KEY. + * + * OpenSolaris OS interface (#ifdefs removed for readability): + * int rijndael_key_setup_dec_intel(uint32_t rk[], + * const uint32_t cipherKey[], uint64_t keyBits); + * int rijndael_key_setup_enc_intel(uint32_t rk[], + * const uint32_t cipherKey[], uint64_t keyBits); + * Return values for above are 0 on error, number of rounds on success. + * + * void aes_encrypt_intel(const aes_ks_t *ks, int Nr, + * const uint32_t pt[4], uint32_t ct[4]); + * void aes_decrypt_intel(const aes_ks_t *ks, int Nr, + * const uint32_t pt[4], uint32_t ct[4]); + * typedef union {uint64_t ks64[(MAX_AES_NR + 1) * 4]; + * uint32_t ks32[(MAX_AES_NR + 1) * 4]; } aes_ks_t; + * + * typedef union { + * uint32_t ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)]; + * } aes_ks_t; + * typedef struct aes_key { + * aes_ks_t encr_ks, decr_ks; + * long double align128; + * int flags, nr, type; + * } aes_key_t; + * + * Note: ks is the AES key schedule, Nr is number of rounds, pt is plain text, + * ct is crypto text, and MAX_AES_NR is 14. + * For the x86 64-bit architecture, OpenSolaris OS uses ks32 instead of ks64. + * + * Note2: aes_ks_t must be aligned on a 0 mod 128 byte boundary. + * + * ==================================================================== + */ + +#if defined(lint) || defined(__lint) + +#include + +/* ARGSUSED */ +void +aes_encrypt_intel(const uint32_t rk[], int Nr, const uint32_t pt[4], + uint32_t ct[4]) { +} +/* ARGSUSED */ +void +aes_decrypt_intel(const uint32_t rk[], int Nr, const uint32_t ct[4], + uint32_t pt[4]) { +} +/* ARGSUSED */ +int +rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[], + uint64_t keyBits) { + return (0); +} +/* ARGSUSED */ +int +rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[], + uint64_t keyBits) { + return (0); +} + + +#else /* lint */ + +#define _ASM +#include + +#ifdef _KERNEL + /* + * Note: the CLTS macro clobbers P2 (%rsi) under i86xpv. That is, + * it calls HYPERVISOR_fpu_taskswitch() which modifies %rsi when it + * uses it to pass P2 to syscall. + * This also occurs with the STTS macro, but we dont care if + * P2 (%rsi) is modified just before function exit. + * The CLTS and STTS macros push and pop P1 (%rdi) already. + */ +#ifdef __xpv +#define PROTECTED_CLTS \ + push %rsi; \ + CLTS; \ + pop %rsi +#else +#define PROTECTED_CLTS \ + CLTS +#endif /* __xpv */ + +#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg) \ + push %rbp; \ + mov %rsp, %rbp; \ + movq %cr0, tmpreg; \ + testq $CR0_TS, tmpreg; \ + jnz 1f; \ + and $-XMM_ALIGN, %rsp; \ + sub $[XMM_SIZE * 2], %rsp; \ + movaps %xmm0, 16(%rsp); \ + movaps %xmm1, (%rsp); \ + jmp 2f; \ +1: \ + PROTECTED_CLTS; \ +2: + + /* + * If CR0_TS was not set above, pop %xmm0 and %xmm1 off stack, + * otherwise set CR0_TS. + */ +#define SET_TS_OR_POP_XMM0_XMM1(tmpreg) \ + testq $CR0_TS, tmpreg; \ + jnz 1f; \ + movaps (%rsp), %xmm1; \ + movaps 16(%rsp), %xmm0; \ + jmp 2f; \ +1: \ + STTS(tmpreg); \ +2: \ + mov %rbp, %rsp; \ + pop %rbp + + /* + * If CR0_TS is not set, align stack (with push %rbp) and push + * %xmm0 - %xmm6 on stack, otherwise clear CR0_TS + */ +#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg) \ + push %rbp; \ + mov %rsp, %rbp; \ + movq %cr0, tmpreg; \ + testq $CR0_TS, tmpreg; \ + jnz 1f; \ + and $-XMM_ALIGN, %rsp; \ + sub $[XMM_SIZE * 7], %rsp; \ + movaps %xmm0, 96(%rsp); \ + movaps %xmm1, 80(%rsp); \ + movaps %xmm2, 64(%rsp); \ + movaps %xmm3, 48(%rsp); \ + movaps %xmm4, 32(%rsp); \ + movaps %xmm5, 16(%rsp); \ + movaps %xmm6, (%rsp); \ + jmp 2f; \ +1: \ + PROTECTED_CLTS; \ +2: + + + /* + * If CR0_TS was not set above, pop %xmm0 - %xmm6 off stack, + * otherwise set CR0_TS. + */ +#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg) \ + testq $CR0_TS, tmpreg; \ + jnz 1f; \ + movaps (%rsp), %xmm6; \ + movaps 16(%rsp), %xmm5; \ + movaps 32(%rsp), %xmm4; \ + movaps 48(%rsp), %xmm3; \ + movaps 64(%rsp), %xmm2; \ + movaps 80(%rsp), %xmm1; \ + movaps 96(%rsp), %xmm0; \ + jmp 2f; \ +1: \ + STTS(tmpreg); \ +2: \ + mov %rbp, %rsp; \ + pop %rbp + + +#else +#define PROTECTED_CLTS +#define CLEAR_TS_OR_PUSH_XMM0_XMM1(tmpreg) +#define SET_TS_OR_POP_XMM0_XMM1(tmpreg) +#define CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(tmpreg) +#define SET_TS_OR_POP_XMM0_TO_XMM6(tmpreg) +#endif /* _KERNEL */ + + +/* + * _key_expansion_128(), * _key_expansion_192a(), _key_expansion_192b(), + * _key_expansion_256a(), _key_expansion_256b() + * + * Helper functions called by rijndael_key_setup_inc_intel(). + * Also used indirectly by rijndael_key_setup_dec_intel(). + * + * Input: + * %xmm0 User-provided cipher key + * %xmm1 Round constant + * Output: + * (%rcx) AES key + */ + +.align 16 +_key_expansion_128: +_key_expansion_256a: + pshufd $0b11111111, %xmm1, %xmm1 + shufps $0b00010000, %xmm0, %xmm4 + pxor %xmm4, %xmm0 + shufps $0b10001100, %xmm0, %xmm4 + pxor %xmm4, %xmm0 + pxor %xmm1, %xmm0 + movaps %xmm0, (%rcx) + add $0x10, %rcx + ret + SET_SIZE(_key_expansion_128) + SET_SIZE(_key_expansion_256a) + +.align 16 +_key_expansion_192a: + pshufd $0b01010101, %xmm1, %xmm1 + shufps $0b00010000, %xmm0, %xmm4 + pxor %xmm4, %xmm0 + shufps $0b10001100, %xmm0, %xmm4 + pxor %xmm4, %xmm0 + pxor %xmm1, %xmm0 + + movaps %xmm2, %xmm5 + movaps %xmm2, %xmm6 + pslldq $4, %xmm5 + pshufd $0b11111111, %xmm0, %xmm3 + pxor %xmm3, %xmm2 + pxor %xmm5, %xmm2 + + movaps %xmm0, %xmm1 + shufps $0b01000100, %xmm0, %xmm6 + movaps %xmm6, (%rcx) + shufps $0b01001110, %xmm2, %xmm1 + movaps %xmm1, 0x10(%rcx) + add $0x20, %rcx + ret + SET_SIZE(_key_expansion_192a) + +.align 16 +_key_expansion_192b: + pshufd $0b01010101, %xmm1, %xmm1 + shufps $0b00010000, %xmm0, %xmm4 + pxor %xmm4, %xmm0 + shufps $0b10001100, %xmm0, %xmm4 + pxor %xmm4, %xmm0 + pxor %xmm1, %xmm0 + + movaps %xmm2, %xmm5 + pslldq $4, %xmm5 + pshufd $0b11111111, %xmm0, %xmm3 + pxor %xmm3, %xmm2 + pxor %xmm5, %xmm2 + + movaps %xmm0, (%rcx) + add $0x10, %rcx + ret + SET_SIZE(_key_expansion_192b) + +.align 16 +_key_expansion_256b: + pshufd $0b10101010, %xmm1, %xmm1 + shufps $0b00010000, %xmm2, %xmm4 + pxor %xmm4, %xmm2 + shufps $0b10001100, %xmm2, %xmm4 + pxor %xmm4, %xmm2 + pxor %xmm1, %xmm2 + movaps %xmm2, (%rcx) + add $0x10, %rcx + ret + SET_SIZE(_key_expansion_256b) + + +/* + * rijndael_key_setup_enc_intel() + * Expand the cipher key into the encryption key schedule. + * + * For kernel code, caller is responsible for ensuring kpreempt_disable() + * has been called. This is because %xmm registers are not saved/restored. + * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set + * on entry. Otherwise, if TS is not set, save and restore %xmm registers + * on the stack. + * + * OpenSolaris interface: + * int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[], + * uint64_t keyBits); + * Return value is 0 on error, number of rounds on success. + * + * Original Intel OpenSSL interface: + * int intel_AES_set_encrypt_key(const unsigned char *userKey, + * const int bits, AES_KEY *key); + * Return value is non-zero on error, 0 on success. + */ + +#ifdef OPENSSL_INTERFACE +#define rijndael_key_setup_enc_intel intel_AES_set_encrypt_key +#define rijndael_key_setup_dec_intel intel_AES_set_decrypt_key + +#define USERCIPHERKEY rdi /* P1, 64 bits */ +#define KEYSIZE32 esi /* P2, 32 bits */ +#define KEYSIZE64 rsi /* P2, 64 bits */ +#define AESKEY rdx /* P3, 64 bits */ + +#else /* OpenSolaris Interface */ +#define AESKEY rdi /* P1, 64 bits */ +#define USERCIPHERKEY rsi /* P2, 64 bits */ +#define KEYSIZE32 edx /* P3, 32 bits */ +#define KEYSIZE64 rdx /* P3, 64 bits */ +#endif /* OPENSSL_INTERFACE */ + +#define ROUNDS32 KEYSIZE32 /* temp */ +#define ROUNDS64 KEYSIZE64 /* temp */ +#define ENDAESKEY USERCIPHERKEY /* temp */ + +ENTRY_NP(rijndael_key_setup_enc_intel) +rijndael_key_setup_enc_intel_local: + CLEAR_TS_OR_PUSH_XMM0_TO_XMM6(%r10) + + // NULL pointer sanity check + test %USERCIPHERKEY, %USERCIPHERKEY + jz .Lenc_key_invalid_param + test %AESKEY, %AESKEY + jz .Lenc_key_invalid_param + + movups (%USERCIPHERKEY), %xmm0 // user key (first 16 bytes) + movaps %xmm0, (%AESKEY) + lea 0x10(%AESKEY), %rcx // key addr + pxor %xmm4, %xmm4 // xmm4 is assumed 0 in _key_expansion_x + + cmp $256, %KEYSIZE32 + jnz .Lenc_key192 + + // AES 256: 14 rounds in encryption key schedule +#ifdef OPENSSL_INTERFACE + mov $14, %ROUNDS32 + movl %ROUNDS32, 240(%AESKEY) // key.rounds = 14 +#endif /* OPENSSL_INTERFACE */ + + movups 0x10(%USERCIPHERKEY), %xmm2 // other user key (2nd 16 bytes) + movaps %xmm2, (%rcx) + add $0x10, %rcx + + aeskeygenassist $0x1, %xmm2, %xmm1 // expand the key + call _key_expansion_256a + aeskeygenassist $0x1, %xmm0, %xmm1 + call _key_expansion_256b + aeskeygenassist $0x2, %xmm2, %xmm1 // expand the key + call _key_expansion_256a + aeskeygenassist $0x2, %xmm0, %xmm1 + call _key_expansion_256b + aeskeygenassist $0x4, %xmm2, %xmm1 // expand the key + call _key_expansion_256a + aeskeygenassist $0x4, %xmm0, %xmm1 + call _key_expansion_256b + aeskeygenassist $0x8, %xmm2, %xmm1 // expand the key + call _key_expansion_256a + aeskeygenassist $0x8, %xmm0, %xmm1 + call _key_expansion_256b + aeskeygenassist $0x10, %xmm2, %xmm1 // expand the key + call _key_expansion_256a + aeskeygenassist $0x10, %xmm0, %xmm1 + call _key_expansion_256b + aeskeygenassist $0x20, %xmm2, %xmm1 // expand the key + call _key_expansion_256a + aeskeygenassist $0x20, %xmm0, %xmm1 + call _key_expansion_256b + aeskeygenassist $0x40, %xmm2, %xmm1 // expand the key + call _key_expansion_256a + + SET_TS_OR_POP_XMM0_TO_XMM6(%r10) +#ifdef OPENSSL_INTERFACE + xor %rax, %rax // return 0 (OK) +#else /* Open Solaris Interface */ + mov $14, %rax // return # rounds = 14 +#endif + ret + +.align 4 +.Lenc_key192: + cmp $192, %KEYSIZE32 + jnz .Lenc_key128 + + // AES 192: 12 rounds in encryption key schedule +#ifdef OPENSSL_INTERFACE + mov $12, %ROUNDS32 + movl %ROUNDS32, 240(%AESKEY) // key.rounds = 12 +#endif /* OPENSSL_INTERFACE */ + + movq 0x10(%USERCIPHERKEY), %xmm2 // other user key + aeskeygenassist $0x1, %xmm2, %xmm1 // expand the key + call _key_expansion_192a + aeskeygenassist $0x2, %xmm2, %xmm1 // expand the key + call _key_expansion_192b + aeskeygenassist $0x4, %xmm2, %xmm1 // expand the key + call _key_expansion_192a + aeskeygenassist $0x8, %xmm2, %xmm1 // expand the key + call _key_expansion_192b + aeskeygenassist $0x10, %xmm2, %xmm1 // expand the key + call _key_expansion_192a + aeskeygenassist $0x20, %xmm2, %xmm1 // expand the key + call _key_expansion_192b + aeskeygenassist $0x40, %xmm2, %xmm1 // expand the key + call _key_expansion_192a + aeskeygenassist $0x80, %xmm2, %xmm1 // expand the key + call _key_expansion_192b + + SET_TS_OR_POP_XMM0_TO_XMM6(%r10) +#ifdef OPENSSL_INTERFACE + xor %rax, %rax // return 0 (OK) +#else /* OpenSolaris Interface */ + mov $12, %rax // return # rounds = 12 +#endif + ret + +.align 4 +.Lenc_key128: + cmp $128, %KEYSIZE32 + jnz .Lenc_key_invalid_key_bits + + // AES 128: 10 rounds in encryption key schedule +#ifdef OPENSSL_INTERFACE + mov $10, %ROUNDS32 + movl %ROUNDS32, 240(%AESKEY) // key.rounds = 10 +#endif /* OPENSSL_INTERFACE */ + + aeskeygenassist $0x1, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + aeskeygenassist $0x2, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + aeskeygenassist $0x4, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + aeskeygenassist $0x8, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + aeskeygenassist $0x10, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + aeskeygenassist $0x20, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + aeskeygenassist $0x40, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + aeskeygenassist $0x80, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + aeskeygenassist $0x1b, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + aeskeygenassist $0x36, %xmm0, %xmm1 // expand the key + call _key_expansion_128 + + SET_TS_OR_POP_XMM0_TO_XMM6(%r10) +#ifdef OPENSSL_INTERFACE + xor %rax, %rax // return 0 (OK) +#else /* OpenSolaris Interface */ + mov $10, %rax // return # rounds = 10 +#endif + ret + +.Lenc_key_invalid_param: +#ifdef OPENSSL_INTERFACE + SET_TS_OR_POP_XMM0_TO_XMM6(%r10) + mov $-1, %rax // user key or AES key pointer is NULL + ret +#else + /* FALLTHROUGH */ +#endif /* OPENSSL_INTERFACE */ + +.Lenc_key_invalid_key_bits: + SET_TS_OR_POP_XMM0_TO_XMM6(%r10) +#ifdef OPENSSL_INTERFACE + mov $-2, %rax // keysize is invalid +#else /* Open Solaris Interface */ + xor %rax, %rax // a key pointer is NULL or invalid keysize +#endif /* OPENSSL_INTERFACE */ + + ret + SET_SIZE(rijndael_key_setup_enc_intel) + + +/* + * rijndael_key_setup_dec_intel() + * Expand the cipher key into the decryption key schedule. + * + * For kernel code, caller is responsible for ensuring kpreempt_disable() + * has been called. This is because %xmm registers are not saved/restored. + * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set + * on entry. Otherwise, if TS is not set, save and restore %xmm registers + * on the stack. + * + * OpenSolaris interface: + * int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[], + * uint64_t keyBits); + * Return value is 0 on error, number of rounds on success. + * P1->P2, P2->P3, P3->P1 + * + * Original Intel OpenSSL interface: + * int intel_AES_set_decrypt_key(const unsigned char *userKey, + * const int bits, AES_KEY *key); + * Return value is non-zero on error, 0 on success. + */ +ENTRY_NP(rijndael_key_setup_dec_intel) + // Generate round keys used for encryption + call rijndael_key_setup_enc_intel_local + test %rax, %rax +#ifdef OPENSSL_INTERFACE + jnz .Ldec_key_exit // Failed if returned non-0 +#else /* OpenSolaris Interface */ + jz .Ldec_key_exit // Failed if returned 0 +#endif /* OPENSSL_INTERFACE */ + + CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10) + + /* + * Convert round keys used for encryption + * to a form usable for decryption + */ +#ifndef OPENSSL_INTERFACE /* OpenSolaris Interface */ + mov %rax, %ROUNDS64 // set # rounds (10, 12, or 14) + // (already set for OpenSSL) +#endif + + lea 0x10(%AESKEY), %rcx // key addr + shl $4, %ROUNDS32 + add %AESKEY, %ROUNDS64 + mov %ROUNDS64, %ENDAESKEY + +.align 4 +.Ldec_key_reorder_loop: + movaps (%AESKEY), %xmm0 + movaps (%ROUNDS64), %xmm1 + movaps %xmm0, (%ROUNDS64) + movaps %xmm1, (%AESKEY) + lea 0x10(%AESKEY), %AESKEY + lea -0x10(%ROUNDS64), %ROUNDS64 + cmp %AESKEY, %ROUNDS64 + ja .Ldec_key_reorder_loop + +.align 4 +.Ldec_key_inv_loop: + movaps (%rcx), %xmm0 + // Convert an encryption round key to a form usable for decryption + // with the "AES Inverse Mix Columns" instruction + aesimc %xmm0, %xmm1 + movaps %xmm1, (%rcx) + lea 0x10(%rcx), %rcx + cmp %ENDAESKEY, %rcx + jnz .Ldec_key_inv_loop + + SET_TS_OR_POP_XMM0_XMM1(%r10) + +.Ldec_key_exit: + // OpenSolaris: rax = # rounds (10, 12, or 14) or 0 for error + // OpenSSL: rax = 0 for OK, or non-zero for error + ret + SET_SIZE(rijndael_key_setup_dec_intel) + + +/* + * aes_encrypt_intel() + * Encrypt a single block (in and out can overlap). + * + * For kernel code, caller is responsible for ensuring kpreempt_disable() + * has been called. This is because %xmm registers are not saved/restored. + * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set + * on entry. Otherwise, if TS is not set, save and restore %xmm registers + * on the stack. + * + * Temporary register usage: + * %xmm0 State + * %xmm1 Key + * + * Original OpenSolaris Interface: + * void aes_encrypt_intel(const aes_ks_t *ks, int Nr, + * const uint32_t pt[4], uint32_t ct[4]) + * + * Original Intel OpenSSL Interface: + * void intel_AES_encrypt(const unsigned char *in, unsigned char *out, + * const AES_KEY *key) + */ + +#ifdef OPENSSL_INTERFACE +#define aes_encrypt_intel intel_AES_encrypt +#define aes_decrypt_intel intel_AES_decrypt + +#define INP rdi /* P1, 64 bits */ +#define OUTP rsi /* P2, 64 bits */ +#define KEYP rdx /* P3, 64 bits */ + +/* No NROUNDS parameter--offset 240 from KEYP saved in %ecx: */ +#define NROUNDS32 ecx /* temporary, 32 bits */ +#define NROUNDS cl /* temporary, 8 bits */ + +#else /* OpenSolaris Interface */ +#define KEYP rdi /* P1, 64 bits */ +#define NROUNDS esi /* P2, 32 bits */ +#define INP rdx /* P3, 64 bits */ +#define OUTP rcx /* P4, 64 bits */ +#endif /* OPENSSL_INTERFACE */ + +#define STATE xmm0 /* temporary, 128 bits */ +#define KEY xmm1 /* temporary, 128 bits */ + +ENTRY_NP(aes_encrypt_intel) + CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10) + + movups (%INP), %STATE // input + movaps (%KEYP), %KEY // key +#ifdef OPENSSL_INTERFACE + mov 240(%KEYP), %NROUNDS32 // round count +#else /* OpenSolaris Interface */ + /* Round count is already present as P2 in %rsi/%esi */ +#endif /* OPENSSL_INTERFACE */ + + pxor %KEY, %STATE // round 0 + lea 0x30(%KEYP), %KEYP + cmp $12, %NROUNDS + jb .Lenc128 + lea 0x20(%KEYP), %KEYP + je .Lenc192 + + // AES 256 + lea 0x20(%KEYP), %KEYP + movaps -0x60(%KEYP), %KEY + aesenc %KEY, %STATE + movaps -0x50(%KEYP), %KEY + aesenc %KEY, %STATE + +.align 4 +.Lenc192: + // AES 192 and 256 + movaps -0x40(%KEYP), %KEY + aesenc %KEY, %STATE + movaps -0x30(%KEYP), %KEY + aesenc %KEY, %STATE + +.align 4 +.Lenc128: + // AES 128, 192, and 256 + movaps -0x20(%KEYP), %KEY + aesenc %KEY, %STATE + movaps -0x10(%KEYP), %KEY + aesenc %KEY, %STATE + movaps (%KEYP), %KEY + aesenc %KEY, %STATE + movaps 0x10(%KEYP), %KEY + aesenc %KEY, %STATE + movaps 0x20(%KEYP), %KEY + aesenc %KEY, %STATE + movaps 0x30(%KEYP), %KEY + aesenc %KEY, %STATE + movaps 0x40(%KEYP), %KEY + aesenc %KEY, %STATE + movaps 0x50(%KEYP), %KEY + aesenc %KEY, %STATE + movaps 0x60(%KEYP), %KEY + aesenc %KEY, %STATE + movaps 0x70(%KEYP), %KEY + aesenclast %KEY, %STATE // last round + movups %STATE, (%OUTP) // output + + SET_TS_OR_POP_XMM0_XMM1(%r10) + ret + SET_SIZE(aes_encrypt_intel) + + +/* + * aes_decrypt_intel() + * Decrypt a single block (in and out can overlap). + * + * For kernel code, caller is responsible for ensuring kpreempt_disable() + * has been called. This is because %xmm registers are not saved/restored. + * Clear and set the CR0.TS bit on entry and exit, respectively, if TS is set + * on entry. Otherwise, if TS is not set, save and restore %xmm registers + * on the stack. + * + * Temporary register usage: + * %xmm0 State + * %xmm1 Key + * + * Original OpenSolaris Interface: + * void aes_decrypt_intel(const aes_ks_t *ks, int Nr, + * const uint32_t pt[4], uint32_t ct[4])/ + * + * Original Intel OpenSSL Interface: + * void intel_AES_decrypt(const unsigned char *in, unsigned char *out, + * const AES_KEY *key); + */ +ENTRY_NP(aes_decrypt_intel) + CLEAR_TS_OR_PUSH_XMM0_XMM1(%r10) + + movups (%INP), %STATE // input + movaps (%KEYP), %KEY // key +#ifdef OPENSSL_INTERFACE + mov 240(%KEYP), %NROUNDS32 // round count +#else /* OpenSolaris Interface */ + /* Round count is already present as P2 in %rsi/%esi */ +#endif /* OPENSSL_INTERFACE */ + + pxor %KEY, %STATE // round 0 + lea 0x30(%KEYP), %KEYP + cmp $12, %NROUNDS + jb .Ldec128 + lea 0x20(%KEYP), %KEYP + je .Ldec192 + + // AES 256 + lea 0x20(%KEYP), %KEYP + movaps -0x60(%KEYP), %KEY + aesdec %KEY, %STATE + movaps -0x50(%KEYP), %KEY + aesdec %KEY, %STATE + +.align 4 +.Ldec192: + // AES 192 and 256 + movaps -0x40(%KEYP), %KEY + aesdec %KEY, %STATE + movaps -0x30(%KEYP), %KEY + aesdec %KEY, %STATE + +.align 4 +.Ldec128: + // AES 128, 192, and 256 + movaps -0x20(%KEYP), %KEY + aesdec %KEY, %STATE + movaps -0x10(%KEYP), %KEY + aesdec %KEY, %STATE + movaps (%KEYP), %KEY + aesdec %KEY, %STATE + movaps 0x10(%KEYP), %KEY + aesdec %KEY, %STATE + movaps 0x20(%KEYP), %KEY + aesdec %KEY, %STATE + movaps 0x30(%KEYP), %KEY + aesdec %KEY, %STATE + movaps 0x40(%KEYP), %KEY + aesdec %KEY, %STATE + movaps 0x50(%KEYP), %KEY + aesdec %KEY, %STATE + movaps 0x60(%KEYP), %KEY + aesdec %KEY, %STATE + movaps 0x70(%KEYP), %KEY + aesdeclast %KEY, %STATE // last round + movups %STATE, (%OUTP) // output + + SET_TS_OR_POP_XMM0_XMM1(%r10) + ret + SET_SIZE(aes_decrypt_intel) + +#endif /* lint || __lint */ diff --git a/module/icp/asm-x86_64/aes/aeskey.c b/module/icp/asm-x86_64/aes/aeskey.c new file mode 100644 index 000000000000..96767fbea06a --- /dev/null +++ b/module/icp/asm-x86_64/aes/aeskey.c @@ -0,0 +1,580 @@ +/* + * --------------------------------------------------------------------------- + * Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved. + * + * LICENSE TERMS + * + * The free distribution and use of this software is allowed (with or without + * changes) provided that: + * + * 1. source code distributions include the above copyright notice, this + * list of conditions and the following disclaimer; + * + * 2. binary distributions include the above copyright notice, this list + * of conditions and the following disclaimer in their documentation; + * + * 3. the name of the copyright holder is not used to endorse products + * built using this software without specific written permission. + * + * DISCLAIMER + * + * This software is provided 'as is' with no explicit or implied warranties + * in respect of its properties, including, but not limited to, correctness + * and/or fitness for purpose. + * --------------------------------------------------------------------------- + * Issue Date: 20/12/2007 + */ + +#include +#include "aesopt.h" +#include "aestab.h" +#include "aestab2.h" + +/* + * Initialise the key schedule from the user supplied key. The key + * length can be specified in bytes, with legal values of 16, 24 + * and 32, or in bits, with legal values of 128, 192 and 256. These + * values correspond with Nk values of 4, 6 and 8 respectively. + * + * The following macros implement a single cycle in the key + * schedule generation process. The number of cycles needed + * for each cx->n_col and nk value is: + * + * nk = 4 5 6 7 8 + * ------------------------------ + * cx->n_col = 4 10 9 8 7 7 + * cx->n_col = 5 14 11 10 9 9 + * cx->n_col = 6 19 15 12 11 11 + * cx->n_col = 7 21 19 16 13 14 + * cx->n_col = 8 29 23 19 17 14 + */ + +/* + * OpenSolaris changes + * 1. Added header files aes_impl.h and aestab2.h + * 2. Changed uint_8t and uint_32t to uint8_t and uint32_t + * 3. Remove code under ifdef USE_VIA_ACE_IF_PRESENT (always undefined) + * 4. Removed always-defined ifdefs FUNCS_IN_C, ENC_KEYING_IN_C, + * AES_128, AES_192, AES_256, AES_VAR defines + * 5. Changed aes_encrypt_key* aes_decrypt_key* functions to "static void" + * 6. Changed N_COLS to MAX_AES_NB + * 7. Replaced functions aes_encrypt_key and aes_decrypt_key with + * OpenSolaris-compatible functions rijndael_key_setup_enc_amd64 and + * rijndael_key_setup_dec_amd64 + * 8. cstyled code and removed lint warnings + */ + +#if defined(REDUCE_CODE_SIZE) +#define ls_box ls_sub + uint32_t ls_sub(const uint32_t t, const uint32_t n); +#define inv_mcol im_sub + uint32_t im_sub(const uint32_t x); +#ifdef ENC_KS_UNROLL +#undef ENC_KS_UNROLL +#endif +#ifdef DEC_KS_UNROLL +#undef DEC_KS_UNROLL +#endif +#endif /* REDUCE_CODE_SIZE */ + + +#define ke4(k, i) \ +{ k[4 * (i) + 4] = ss[0] ^= ls_box(ss[3], 3) ^ t_use(r, c)[i]; \ + k[4 * (i) + 5] = ss[1] ^= ss[0]; \ + k[4 * (i) + 6] = ss[2] ^= ss[1]; \ + k[4 * (i) + 7] = ss[3] ^= ss[2]; \ +} + +static void +aes_encrypt_key128(const unsigned char *key, uint32_t rk[]) +{ + uint32_t ss[4]; + + rk[0] = ss[0] = word_in(key, 0); + rk[1] = ss[1] = word_in(key, 1); + rk[2] = ss[2] = word_in(key, 2); + rk[3] = ss[3] = word_in(key, 3); + +#ifdef ENC_KS_UNROLL + ke4(rk, 0); ke4(rk, 1); + ke4(rk, 2); ke4(rk, 3); + ke4(rk, 4); ke4(rk, 5); + ke4(rk, 6); ke4(rk, 7); + ke4(rk, 8); +#else + { + uint32_t i; + for (i = 0; i < 9; ++i) + ke4(rk, i); + } +#endif /* ENC_KS_UNROLL */ + ke4(rk, 9); +} + + +#define kef6(k, i) \ +{ k[6 * (i) + 6] = ss[0] ^= ls_box(ss[5], 3) ^ t_use(r, c)[i]; \ + k[6 * (i) + 7] = ss[1] ^= ss[0]; \ + k[6 * (i) + 8] = ss[2] ^= ss[1]; \ + k[6 * (i) + 9] = ss[3] ^= ss[2]; \ +} + +#define ke6(k, i) \ +{ kef6(k, i); \ + k[6 * (i) + 10] = ss[4] ^= ss[3]; \ + k[6 * (i) + 11] = ss[5] ^= ss[4]; \ +} + +static void +aes_encrypt_key192(const unsigned char *key, uint32_t rk[]) +{ + uint32_t ss[6]; + + rk[0] = ss[0] = word_in(key, 0); + rk[1] = ss[1] = word_in(key, 1); + rk[2] = ss[2] = word_in(key, 2); + rk[3] = ss[3] = word_in(key, 3); + rk[4] = ss[4] = word_in(key, 4); + rk[5] = ss[5] = word_in(key, 5); + +#ifdef ENC_KS_UNROLL + ke6(rk, 0); ke6(rk, 1); + ke6(rk, 2); ke6(rk, 3); + ke6(rk, 4); ke6(rk, 5); + ke6(rk, 6); +#else + { + uint32_t i; + for (i = 0; i < 7; ++i) + ke6(rk, i); + } +#endif /* ENC_KS_UNROLL */ + kef6(rk, 7); +} + + + +#define kef8(k, i) \ +{ k[8 * (i) + 8] = ss[0] ^= ls_box(ss[7], 3) ^ t_use(r, c)[i]; \ + k[8 * (i) + 9] = ss[1] ^= ss[0]; \ + k[8 * (i) + 10] = ss[2] ^= ss[1]; \ + k[8 * (i) + 11] = ss[3] ^= ss[2]; \ +} + +#define ke8(k, i) \ +{ kef8(k, i); \ + k[8 * (i) + 12] = ss[4] ^= ls_box(ss[3], 0); \ + k[8 * (i) + 13] = ss[5] ^= ss[4]; \ + k[8 * (i) + 14] = ss[6] ^= ss[5]; \ + k[8 * (i) + 15] = ss[7] ^= ss[6]; \ +} + +static void +aes_encrypt_key256(const unsigned char *key, uint32_t rk[]) +{ + uint32_t ss[8]; + + rk[0] = ss[0] = word_in(key, 0); + rk[1] = ss[1] = word_in(key, 1); + rk[2] = ss[2] = word_in(key, 2); + rk[3] = ss[3] = word_in(key, 3); + rk[4] = ss[4] = word_in(key, 4); + rk[5] = ss[5] = word_in(key, 5); + rk[6] = ss[6] = word_in(key, 6); + rk[7] = ss[7] = word_in(key, 7); + +#ifdef ENC_KS_UNROLL + ke8(rk, 0); ke8(rk, 1); + ke8(rk, 2); ke8(rk, 3); + ke8(rk, 4); ke8(rk, 5); +#else + { + uint32_t i; + for (i = 0; i < 6; ++i) + ke8(rk, i); + } +#endif /* ENC_KS_UNROLL */ + kef8(rk, 6); +} + + +/* + * Expand the cipher key into the encryption key schedule. + * + * Return the number of rounds for the given cipher key size. + * The size of the key schedule depends on the number of rounds + * (which can be computed from the size of the key), i.e. 4 * (Nr + 1). + * + * Parameters: + * rk AES key schedule 32-bit array to be initialized + * cipherKey User key + * keyBits AES key size (128, 192, or 256 bits) + */ +int +rijndael_key_setup_enc_amd64(uint32_t rk[], const uint32_t cipherKey[], + int keyBits) +{ + switch (keyBits) { + case 128: + aes_encrypt_key128((unsigned char *)&cipherKey[0], rk); + return (10); + case 192: + aes_encrypt_key192((unsigned char *)&cipherKey[0], rk); + return (12); + case 256: + aes_encrypt_key256((unsigned char *)&cipherKey[0], rk); + return (14); + default: /* should never get here */ + break; + } + + return (0); +} + + +/* this is used to store the decryption round keys */ +/* in forward or reverse order */ + +#ifdef AES_REV_DKS +#define v(n, i) ((n) - (i) + 2 * ((i) & 3)) +#else +#define v(n, i) (i) +#endif + +#if DEC_ROUND == NO_TABLES +#define ff(x) (x) +#else +#define ff(x) inv_mcol(x) +#if defined(dec_imvars) +#define d_vars dec_imvars +#endif +#endif /* FUNCS_IN_C & DEC_KEYING_IN_C */ + + +#define k4e(k, i) \ +{ k[v(40, (4 * (i)) + 4)] = ss[0] ^= ls_box(ss[3], 3) ^ t_use(r, c)[i]; \ + k[v(40, (4 * (i)) + 5)] = ss[1] ^= ss[0]; \ + k[v(40, (4 * (i)) + 6)] = ss[2] ^= ss[1]; \ + k[v(40, (4 * (i)) + 7)] = ss[3] ^= ss[2]; \ +} + +#if 1 + +#define kdf4(k, i) \ +{ ss[0] = ss[0] ^ ss[2] ^ ss[1] ^ ss[3]; \ + ss[1] = ss[1] ^ ss[3]; \ + ss[2] = ss[2] ^ ss[3]; \ + ss[4] = ls_box(ss[(i + 3) % 4], 3) ^ t_use(r, c)[i]; \ + ss[i % 4] ^= ss[4]; \ + ss[4] ^= k[v(40, (4 * (i)))]; k[v(40, (4 * (i)) + 4)] = ff(ss[4]); \ + ss[4] ^= k[v(40, (4 * (i)) + 1)]; k[v(40, (4 * (i)) + 5)] = ff(ss[4]); \ + ss[4] ^= k[v(40, (4 * (i)) + 2)]; k[v(40, (4 * (i)) + 6)] = ff(ss[4]); \ + ss[4] ^= k[v(40, (4 * (i)) + 3)]; k[v(40, (4 * (i)) + 7)] = ff(ss[4]); \ +} + +#define kd4(k, i) \ +{ ss[4] = ls_box(ss[(i + 3) % 4], 3) ^ t_use(r, c)[i]; \ + ss[i % 4] ^= ss[4]; ss[4] = ff(ss[4]); \ + k[v(40, (4 * (i)) + 4)] = ss[4] ^= k[v(40, (4 * (i)))]; \ + k[v(40, (4 * (i)) + 5)] = ss[4] ^= k[v(40, (4 * (i)) + 1)]; \ + k[v(40, (4 * (i)) + 6)] = ss[4] ^= k[v(40, (4 * (i)) + 2)]; \ + k[v(40, (4 * (i)) + 7)] = ss[4] ^= k[v(40, (4 * (i)) + 3)]; \ +} + +#define kdl4(k, i) \ +{ ss[4] = ls_box(ss[(i + 3) % 4], 3) ^ t_use(r, c)[i]; \ + ss[i % 4] ^= ss[4]; \ + k[v(40, (4 * (i)) + 4)] = (ss[0] ^= ss[1]) ^ ss[2] ^ ss[3]; \ + k[v(40, (4 * (i)) + 5)] = ss[1] ^ ss[3]; \ + k[v(40, (4 * (i)) + 6)] = ss[0]; \ + k[v(40, (4 * (i)) + 7)] = ss[1]; \ +} + +#else + +#define kdf4(k, i) \ +{ ss[0] ^= ls_box(ss[3], 3) ^ t_use(r, c)[i]; \ + k[v(40, (4 * (i)) + 4)] = ff(ss[0]); \ + ss[1] ^= ss[0]; k[v(40, (4 * (i)) + 5)] = ff(ss[1]); \ + ss[2] ^= ss[1]; k[v(40, (4 * (i)) + 6)] = ff(ss[2]); \ + ss[3] ^= ss[2]; k[v(40, (4 * (i)) + 7)] = ff(ss[3]); \ +} + +#define kd4(k, i) \ +{ ss[4] = ls_box(ss[3], 3) ^ t_use(r, c)[i]; \ + ss[0] ^= ss[4]; \ + ss[4] = ff(ss[4]); \ + k[v(40, (4 * (i)) + 4)] = ss[4] ^= k[v(40, (4 * (i)))]; \ + ss[1] ^= ss[0]; \ + k[v(40, (4 * (i)) + 5)] = ss[4] ^= k[v(40, (4 * (i)) + 1)]; \ + ss[2] ^= ss[1]; \ + k[v(40, (4 * (i)) + 6)] = ss[4] ^= k[v(40, (4 * (i)) + 2)]; \ + ss[3] ^= ss[2]; \ + k[v(40, (4 * (i)) + 7)] = ss[4] ^= k[v(40, (4 * (i)) + 3)]; \ +} + +#define kdl4(k, i) \ +{ ss[0] ^= ls_box(ss[3], 3) ^ t_use(r, c)[i]; \ + k[v(40, (4 * (i)) + 4)] = ss[0]; \ + ss[1] ^= ss[0]; k[v(40, (4 * (i)) + 5)] = ss[1]; \ + ss[2] ^= ss[1]; k[v(40, (4 * (i)) + 6)] = ss[2]; \ + ss[3] ^= ss[2]; k[v(40, (4 * (i)) + 7)] = ss[3]; \ +} + +#endif + +static void +aes_decrypt_key128(const unsigned char *key, uint32_t rk[]) +{ + uint32_t ss[5]; +#if defined(d_vars) + d_vars; +#endif + rk[v(40, (0))] = ss[0] = word_in(key, 0); + rk[v(40, (1))] = ss[1] = word_in(key, 1); + rk[v(40, (2))] = ss[2] = word_in(key, 2); + rk[v(40, (3))] = ss[3] = word_in(key, 3); + +#ifdef DEC_KS_UNROLL + kdf4(rk, 0); kd4(rk, 1); + kd4(rk, 2); kd4(rk, 3); + kd4(rk, 4); kd4(rk, 5); + kd4(rk, 6); kd4(rk, 7); + kd4(rk, 8); kdl4(rk, 9); +#else + { + uint32_t i; + for (i = 0; i < 10; ++i) + k4e(rk, i); +#if !(DEC_ROUND == NO_TABLES) + for (i = MAX_AES_NB; i < 10 * MAX_AES_NB; ++i) + rk[i] = inv_mcol(rk[i]); +#endif + } +#endif /* DEC_KS_UNROLL */ +} + + + +#define k6ef(k, i) \ +{ k[v(48, (6 * (i)) + 6)] = ss[0] ^= ls_box(ss[5], 3) ^ t_use(r, c)[i]; \ + k[v(48, (6 * (i)) + 7)] = ss[1] ^= ss[0]; \ + k[v(48, (6 * (i)) + 8)] = ss[2] ^= ss[1]; \ + k[v(48, (6 * (i)) + 9)] = ss[3] ^= ss[2]; \ +} + +#define k6e(k, i) \ +{ k6ef(k, i); \ + k[v(48, (6 * (i)) + 10)] = ss[4] ^= ss[3]; \ + k[v(48, (6 * (i)) + 11)] = ss[5] ^= ss[4]; \ +} + +#define kdf6(k, i) \ +{ ss[0] ^= ls_box(ss[5], 3) ^ t_use(r, c)[i]; \ + k[v(48, (6 * (i)) + 6)] = ff(ss[0]); \ + ss[1] ^= ss[0]; k[v(48, (6 * (i)) + 7)] = ff(ss[1]); \ + ss[2] ^= ss[1]; k[v(48, (6 * (i)) + 8)] = ff(ss[2]); \ + ss[3] ^= ss[2]; k[v(48, (6 * (i)) + 9)] = ff(ss[3]); \ + ss[4] ^= ss[3]; k[v(48, (6 * (i)) + 10)] = ff(ss[4]); \ + ss[5] ^= ss[4]; k[v(48, (6 * (i)) + 11)] = ff(ss[5]); \ +} + +#define kd6(k, i) \ +{ ss[6] = ls_box(ss[5], 3) ^ t_use(r, c)[i]; \ + ss[0] ^= ss[6]; ss[6] = ff(ss[6]); \ + k[v(48, (6 * (i)) + 6)] = ss[6] ^= k[v(48, (6 * (i)))]; \ + ss[1] ^= ss[0]; \ + k[v(48, (6 * (i)) + 7)] = ss[6] ^= k[v(48, (6 * (i)) + 1)]; \ + ss[2] ^= ss[1]; \ + k[v(48, (6 * (i)) + 8)] = ss[6] ^= k[v(48, (6 * (i)) + 2)]; \ + ss[3] ^= ss[2]; \ + k[v(48, (6 * (i)) + 9)] = ss[6] ^= k[v(48, (6 * (i)) + 3)]; \ + ss[4] ^= ss[3]; \ + k[v(48, (6 * (i)) + 10)] = ss[6] ^= k[v(48, (6 * (i)) + 4)]; \ + ss[5] ^= ss[4]; \ + k[v(48, (6 * (i)) + 11)] = ss[6] ^= k[v(48, (6 * (i)) + 5)]; \ +} + +#define kdl6(k, i) \ +{ ss[0] ^= ls_box(ss[5], 3) ^ t_use(r, c)[i]; \ + k[v(48, (6 * (i)) + 6)] = ss[0]; \ + ss[1] ^= ss[0]; k[v(48, (6 * (i)) + 7)] = ss[1]; \ + ss[2] ^= ss[1]; k[v(48, (6 * (i)) + 8)] = ss[2]; \ + ss[3] ^= ss[2]; k[v(48, (6 * (i)) + 9)] = ss[3]; \ +} + +static void +aes_decrypt_key192(const unsigned char *key, uint32_t rk[]) +{ + uint32_t ss[7]; +#if defined(d_vars) + d_vars; +#endif + rk[v(48, (0))] = ss[0] = word_in(key, 0); + rk[v(48, (1))] = ss[1] = word_in(key, 1); + rk[v(48, (2))] = ss[2] = word_in(key, 2); + rk[v(48, (3))] = ss[3] = word_in(key, 3); + +#ifdef DEC_KS_UNROLL + ss[4] = word_in(key, 4); + rk[v(48, (4))] = ff(ss[4]); + ss[5] = word_in(key, 5); + rk[v(48, (5))] = ff(ss[5]); + kdf6(rk, 0); kd6(rk, 1); + kd6(rk, 2); kd6(rk, 3); + kd6(rk, 4); kd6(rk, 5); + kd6(rk, 6); kdl6(rk, 7); +#else + rk[v(48, (4))] = ss[4] = word_in(key, 4); + rk[v(48, (5))] = ss[5] = word_in(key, 5); + { + uint32_t i; + + for (i = 0; i < 7; ++i) + k6e(rk, i); + k6ef(rk, 7); +#if !(DEC_ROUND == NO_TABLES) + for (i = MAX_AES_NB; i < 12 * MAX_AES_NB; ++i) + rk[i] = inv_mcol(rk[i]); +#endif + } +#endif +} + + + +#define k8ef(k, i) \ +{ k[v(56, (8 * (i)) + 8)] = ss[0] ^= ls_box(ss[7], 3) ^ t_use(r, c)[i]; \ + k[v(56, (8 * (i)) + 9)] = ss[1] ^= ss[0]; \ + k[v(56, (8 * (i)) + 10)] = ss[2] ^= ss[1]; \ + k[v(56, (8 * (i)) + 11)] = ss[3] ^= ss[2]; \ +} + +#define k8e(k, i) \ +{ k8ef(k, i); \ + k[v(56, (8 * (i)) + 12)] = ss[4] ^= ls_box(ss[3], 0); \ + k[v(56, (8 * (i)) + 13)] = ss[5] ^= ss[4]; \ + k[v(56, (8 * (i)) + 14)] = ss[6] ^= ss[5]; \ + k[v(56, (8 * (i)) + 15)] = ss[7] ^= ss[6]; \ +} + +#define kdf8(k, i) \ +{ ss[0] ^= ls_box(ss[7], 3) ^ t_use(r, c)[i]; \ + k[v(56, (8 * (i)) + 8)] = ff(ss[0]); \ + ss[1] ^= ss[0]; k[v(56, (8 * (i)) + 9)] = ff(ss[1]); \ + ss[2] ^= ss[1]; k[v(56, (8 * (i)) + 10)] = ff(ss[2]); \ + ss[3] ^= ss[2]; k[v(56, (8 * (i)) + 11)] = ff(ss[3]); \ + ss[4] ^= ls_box(ss[3], 0); k[v(56, (8 * (i)) + 12)] = ff(ss[4]); \ + ss[5] ^= ss[4]; k[v(56, (8 * (i)) + 13)] = ff(ss[5]); \ + ss[6] ^= ss[5]; k[v(56, (8 * (i)) + 14)] = ff(ss[6]); \ + ss[7] ^= ss[6]; k[v(56, (8 * (i)) + 15)] = ff(ss[7]); \ +} + +#define kd8(k, i) \ +{ ss[8] = ls_box(ss[7], 3) ^ t_use(r, c)[i]; \ + ss[0] ^= ss[8]; \ + ss[8] = ff(ss[8]); \ + k[v(56, (8 * (i)) + 8)] = ss[8] ^= k[v(56, (8 * (i)))]; \ + ss[1] ^= ss[0]; \ + k[v(56, (8 * (i)) + 9)] = ss[8] ^= k[v(56, (8 * (i)) + 1)]; \ + ss[2] ^= ss[1]; \ + k[v(56, (8 * (i)) + 10)] = ss[8] ^= k[v(56, (8 * (i)) + 2)]; \ + ss[3] ^= ss[2]; \ + k[v(56, (8 * (i)) + 11)] = ss[8] ^= k[v(56, (8 * (i)) + 3)]; \ + ss[8] = ls_box(ss[3], 0); \ + ss[4] ^= ss[8]; \ + ss[8] = ff(ss[8]); \ + k[v(56, (8 * (i)) + 12)] = ss[8] ^= k[v(56, (8 * (i)) + 4)]; \ + ss[5] ^= ss[4]; \ + k[v(56, (8 * (i)) + 13)] = ss[8] ^= k[v(56, (8 * (i)) + 5)]; \ + ss[6] ^= ss[5]; \ + k[v(56, (8 * (i)) + 14)] = ss[8] ^= k[v(56, (8 * (i)) + 6)]; \ + ss[7] ^= ss[6]; \ + k[v(56, (8 * (i)) + 15)] = ss[8] ^= k[v(56, (8 * (i)) + 7)]; \ +} + +#define kdl8(k, i) \ +{ ss[0] ^= ls_box(ss[7], 3) ^ t_use(r, c)[i]; \ + k[v(56, (8 * (i)) + 8)] = ss[0]; \ + ss[1] ^= ss[0]; k[v(56, (8 * (i)) + 9)] = ss[1]; \ + ss[2] ^= ss[1]; k[v(56, (8 * (i)) + 10)] = ss[2]; \ + ss[3] ^= ss[2]; k[v(56, (8 * (i)) + 11)] = ss[3]; \ +} + +static void +aes_decrypt_key256(const unsigned char *key, uint32_t rk[]) +{ + uint32_t ss[9]; +#if defined(d_vars) + d_vars; +#endif + rk[v(56, (0))] = ss[0] = word_in(key, 0); + rk[v(56, (1))] = ss[1] = word_in(key, 1); + rk[v(56, (2))] = ss[2] = word_in(key, 2); + rk[v(56, (3))] = ss[3] = word_in(key, 3); + +#ifdef DEC_KS_UNROLL + ss[4] = word_in(key, 4); + rk[v(56, (4))] = ff(ss[4]); + ss[5] = word_in(key, 5); + rk[v(56, (5))] = ff(ss[5]); + ss[6] = word_in(key, 6); + rk[v(56, (6))] = ff(ss[6]); + ss[7] = word_in(key, 7); + rk[v(56, (7))] = ff(ss[7]); + kdf8(rk, 0); kd8(rk, 1); + kd8(rk, 2); kd8(rk, 3); + kd8(rk, 4); kd8(rk, 5); + kdl8(rk, 6); +#else + rk[v(56, (4))] = ss[4] = word_in(key, 4); + rk[v(56, (5))] = ss[5] = word_in(key, 5); + rk[v(56, (6))] = ss[6] = word_in(key, 6); + rk[v(56, (7))] = ss[7] = word_in(key, 7); + { + uint32_t i; + + for (i = 0; i < 6; ++i) + k8e(rk, i); + k8ef(rk, 6); +#if !(DEC_ROUND == NO_TABLES) + for (i = MAX_AES_NB; i < 14 * MAX_AES_NB; ++i) + rk[i] = inv_mcol(rk[i]); +#endif + } +#endif /* DEC_KS_UNROLL */ +} + + +/* + * Expand the cipher key into the decryption key schedule. + * + * Return the number of rounds for the given cipher key size. + * The size of the key schedule depends on the number of rounds + * (which can be computed from the size of the key), i.e. 4 * (Nr + 1). + * + * Parameters: + * rk AES key schedule 32-bit array to be initialized + * cipherKey User key + * keyBits AES key size (128, 192, or 256 bits) + */ +int +rijndael_key_setup_dec_amd64(uint32_t rk[], const uint32_t cipherKey[], + int keyBits) +{ + switch (keyBits) { + case 128: + aes_decrypt_key128((unsigned char *)&cipherKey[0], rk); + return (10); + case 192: + aes_decrypt_key192((unsigned char *)&cipherKey[0], rk); + return (12); + case 256: + aes_decrypt_key256((unsigned char *)&cipherKey[0], rk); + return (14); + default: /* should never get here */ + break; + } + + return (0); +} diff --git a/module/icp/asm-x86_64/aes/aesopt.h b/module/icp/asm-x86_64/aes/aesopt.h new file mode 100644 index 000000000000..6aa61db8275a --- /dev/null +++ b/module/icp/asm-x86_64/aes/aesopt.h @@ -0,0 +1,770 @@ +/* + * --------------------------------------------------------------------------- + * Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved. + * + * LICENSE TERMS + * + * The free distribution and use of this software is allowed (with or without + * changes) provided that: + * + * 1. source code distributions include the above copyright notice, this + * list of conditions and the following disclaimer; + * + * 2. binary distributions include the above copyright notice, this list + * of conditions and the following disclaimer in their documentation; + * + * 3. the name of the copyright holder is not used to endorse products + * built using this software without specific written permission. + * + * DISCLAIMER + * + * This software is provided 'as is' with no explicit or implied warranties + * in respect of its properties, including, but not limited to, correctness + * and/or fitness for purpose. + * --------------------------------------------------------------------------- + * Issue Date: 20/12/2007 + * + * This file contains the compilation options for AES (Rijndael) and code + * that is common across encryption, key scheduling and table generation. + * + * OPERATION + * + * These source code files implement the AES algorithm Rijndael designed by + * Joan Daemen and Vincent Rijmen. This version is designed for the standard + * block size of 16 bytes and for key sizes of 128, 192 and 256 bits (16, 24 + * and 32 bytes). + * + * This version is designed for flexibility and speed using operations on + * 32-bit words rather than operations on bytes. It can be compiled with + * either big or little endian internal byte order but is faster when the + * native byte order for the processor is used. + * + * THE CIPHER INTERFACE + * + * The cipher interface is implemented as an array of bytes in which lower + * AES bit sequence indexes map to higher numeric significance within bytes. + */ + +/* + * OpenSolaris changes + * 1. Added __cplusplus and _AESTAB_H header guards + * 2. Added header files sys/types.h and aes_impl.h + * 3. Added defines for AES_ENCRYPT, AES_DECRYPT, AES_REV_DKS, and ASM_AMD64_C + * 4. Moved defines for IS_BIG_ENDIAN, IS_LITTLE_ENDIAN, PLATFORM_BYTE_ORDER + * from brg_endian.h + * 5. Undefined VIA_ACE_POSSIBLE and ASSUME_VIA_ACE_PRESENT + * 6. Changed uint_8t and uint_32t to uint8_t and uint32_t + * 7. Defined aes_sw32 as htonl() for byte swapping + * 8. Cstyled and hdrchk code + * + */ + +#ifndef _AESOPT_H +#define _AESOPT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +/* SUPPORT FEATURES */ +#define AES_ENCRYPT /* if support for encryption is needed */ +#define AES_DECRYPT /* if support for decryption is needed */ + +/* PLATFORM-SPECIFIC FEATURES */ +#define IS_BIG_ENDIAN 4321 /* byte 0 is most significant (mc68k) */ +#define IS_LITTLE_ENDIAN 1234 /* byte 0 is least significant (i386) */ +#define PLATFORM_BYTE_ORDER IS_LITTLE_ENDIAN +#define AES_REV_DKS /* define to reverse decryption key schedule */ + + +/* + * CONFIGURATION - THE USE OF DEFINES + * Later in this section there are a number of defines that control the + * operation of the code. In each section, the purpose of each define is + * explained so that the relevant form can be included or excluded by + * setting either 1's or 0's respectively on the branches of the related + * #if clauses. The following local defines should not be changed. + */ + +#define ENCRYPTION_IN_C 1 +#define DECRYPTION_IN_C 2 +#define ENC_KEYING_IN_C 4 +#define DEC_KEYING_IN_C 8 + +#define NO_TABLES 0 +#define ONE_TABLE 1 +#define FOUR_TABLES 4 +#define NONE 0 +#define PARTIAL 1 +#define FULL 2 + +/* --- START OF USER CONFIGURED OPTIONS --- */ + +/* + * 1. BYTE ORDER WITHIN 32 BIT WORDS + * + * The fundamental data processing units in Rijndael are 8-bit bytes. The + * input, output and key input are all enumerated arrays of bytes in which + * bytes are numbered starting at zero and increasing to one less than the + * number of bytes in the array in question. This enumeration is only used + * for naming bytes and does not imply any adjacency or order relationship + * from one byte to another. When these inputs and outputs are considered + * as bit sequences, bits 8*n to 8*n+7 of the bit sequence are mapped to + * byte[n] with bit 8n+i in the sequence mapped to bit 7-i within the byte. + * In this implementation bits are numbered from 0 to 7 starting at the + * numerically least significant end of each byte. Bit n represents 2^n. + * + * However, Rijndael can be implemented more efficiently using 32-bit + * words by packing bytes into words so that bytes 4*n to 4*n+3 are placed + * into word[n]. While in principle these bytes can be assembled into words + * in any positions, this implementation only supports the two formats in + * which bytes in adjacent positions within words also have adjacent byte + * numbers. This order is called big-endian if the lowest numbered bytes + * in words have the highest numeric significance and little-endian if the + * opposite applies. + * + * This code can work in either order irrespective of the order used by the + * machine on which it runs. Normally the internal byte order will be set + * to the order of the processor on which the code is to be run but this + * define can be used to reverse this in special situations + * + * WARNING: Assembler code versions rely on PLATFORM_BYTE_ORDER being set. + * This define will hence be redefined later (in section 4) if necessary + */ + +#if 1 +#define ALGORITHM_BYTE_ORDER PLATFORM_BYTE_ORDER +#elif 0 +#define ALGORITHM_BYTE_ORDER IS_LITTLE_ENDIAN +#elif 0 +#define ALGORITHM_BYTE_ORDER IS_BIG_ENDIAN +#else +#error The algorithm byte order is not defined +#endif + +/* 2. VIA ACE SUPPORT */ + +#if defined(__GNUC__) && defined(__i386__) || \ + defined(_WIN32) && defined(_M_IX86) && \ + !(defined(_WIN64) || defined(_WIN32_WCE) || \ + defined(_MSC_VER) && (_MSC_VER <= 800)) +#define VIA_ACE_POSSIBLE +#endif + +/* + * Define this option if support for the VIA ACE is required. This uses + * inline assembler instructions and is only implemented for the Microsoft, + * Intel and GCC compilers. If VIA ACE is known to be present, then defining + * ASSUME_VIA_ACE_PRESENT will remove the ordinary encryption/decryption + * code. If USE_VIA_ACE_IF_PRESENT is defined then VIA ACE will be used if + * it is detected (both present and enabled) but the normal AES code will + * also be present. + * + * When VIA ACE is to be used, all AES encryption contexts MUST be 16 byte + * aligned; other input/output buffers do not need to be 16 byte aligned + * but there are very large performance gains if this can be arranged. + * VIA ACE also requires the decryption key schedule to be in reverse + * order (which later checks below ensure). + */ + +/* VIA ACE is not used here for OpenSolaris: */ +#undef VIA_ACE_POSSIBLE +#undef ASSUME_VIA_ACE_PRESENT + +#if 0 && defined(VIA_ACE_POSSIBLE) && !defined(USE_VIA_ACE_IF_PRESENT) +#define USE_VIA_ACE_IF_PRESENT +#endif + +#if 0 && defined(VIA_ACE_POSSIBLE) && !defined(ASSUME_VIA_ACE_PRESENT) +#define ASSUME_VIA_ACE_PRESENT +#endif + + +/* + * 3. ASSEMBLER SUPPORT + * + * This define (which can be on the command line) enables the use of the + * assembler code routines for encryption, decryption and key scheduling + * as follows: + * + * ASM_X86_V1C uses the assembler (aes_x86_v1.asm) with large tables for + * encryption and decryption and but with key scheduling in C + * ASM_X86_V2 uses assembler (aes_x86_v2.asm) with compressed tables for + * encryption, decryption and key scheduling + * ASM_X86_V2C uses assembler (aes_x86_v2.asm) with compressed tables for + * encryption and decryption and but with key scheduling in C + * ASM_AMD64_C uses assembler (aes_amd64.asm) with compressed tables for + * encryption and decryption and but with key scheduling in C + * + * Change one 'if 0' below to 'if 1' to select the version or define + * as a compilation option. + */ + +#if 0 && !defined(ASM_X86_V1C) +#define ASM_X86_V1C +#elif 0 && !defined(ASM_X86_V2) +#define ASM_X86_V2 +#elif 0 && !defined(ASM_X86_V2C) +#define ASM_X86_V2C +#elif 1 && !defined(ASM_AMD64_C) +#define ASM_AMD64_C +#endif + +#if (defined(ASM_X86_V1C) || defined(ASM_X86_V2) || defined(ASM_X86_V2C)) && \ + !defined(_M_IX86) || defined(ASM_AMD64_C) && !defined(_M_X64) && \ + !defined(__amd64) +#error Assembler code is only available for x86 and AMD64 systems +#endif + +/* + * 4. FAST INPUT/OUTPUT OPERATIONS. + * + * On some machines it is possible to improve speed by transferring the + * bytes in the input and output arrays to and from the internal 32-bit + * variables by addressing these arrays as if they are arrays of 32-bit + * words. On some machines this will always be possible but there may + * be a large performance penalty if the byte arrays are not aligned on + * the normal word boundaries. On other machines this technique will + * lead to memory access errors when such 32-bit word accesses are not + * properly aligned. The option SAFE_IO avoids such problems but will + * often be slower on those machines that support misaligned access + * (especially so if care is taken to align the input and output byte + * arrays on 32-bit word boundaries). If SAFE_IO is not defined it is + * assumed that access to byte arrays as if they are arrays of 32-bit + * words will not cause problems when such accesses are misaligned. + */ +#if 1 && !defined(_MSC_VER) +#define SAFE_IO +#endif + +/* + * 5. LOOP UNROLLING + * + * The code for encryption and decryption cycles through a number of rounds + * that can be implemented either in a loop or by expanding the code into a + * long sequence of instructions, the latter producing a larger program but + * one that will often be much faster. The latter is called loop unrolling. + * There are also potential speed advantages in expanding two iterations in + * a loop with half the number of iterations, which is called partial loop + * unrolling. The following options allow partial or full loop unrolling + * to be set independently for encryption and decryption + */ +#if 1 +#define ENC_UNROLL FULL +#elif 0 +#define ENC_UNROLL PARTIAL +#else +#define ENC_UNROLL NONE +#endif + +#if 1 +#define DEC_UNROLL FULL +#elif 0 +#define DEC_UNROLL PARTIAL +#else +#define DEC_UNROLL NONE +#endif + +#if 1 +#define ENC_KS_UNROLL +#endif + +#if 1 +#define DEC_KS_UNROLL +#endif + +/* + * 6. FAST FINITE FIELD OPERATIONS + * + * If this section is included, tables are used to provide faster finite + * field arithmetic. This has no effect if FIXED_TABLES is defined. + */ +#if 1 +#define FF_TABLES +#endif + +/* + * 7. INTERNAL STATE VARIABLE FORMAT + * + * The internal state of Rijndael is stored in a number of local 32-bit + * word variables which can be defined either as an array or as individual + * names variables. Include this section if you want to store these local + * variables in arrays. Otherwise individual local variables will be used. + */ +#if 1 +#define ARRAYS +#endif + +/* + * 8. FIXED OR DYNAMIC TABLES + * + * When this section is included the tables used by the code are compiled + * statically into the binary file. Otherwise the subroutine aes_init() + * must be called to compute them before the code is first used. + */ +#if 1 && !(defined(_MSC_VER) && (_MSC_VER <= 800)) +#define FIXED_TABLES +#endif + +/* + * 9. MASKING OR CASTING FROM LONGER VALUES TO BYTES + * + * In some systems it is better to mask longer values to extract bytes + * rather than using a cast. This option allows this choice. + */ +#if 0 +#define to_byte(x) ((uint8_t)(x)) +#else +#define to_byte(x) ((x) & 0xff) +#endif + +/* + * 10. TABLE ALIGNMENT + * + * On some systems speed will be improved by aligning the AES large lookup + * tables on particular boundaries. This define should be set to a power of + * two giving the desired alignment. It can be left undefined if alignment + * is not needed. This option is specific to the Micrsoft VC++ compiler - + * it seems to sometimes cause trouble for the VC++ version 6 compiler. + */ + +#if 1 && defined(_MSC_VER) && (_MSC_VER >= 1300) +#define TABLE_ALIGN 32 +#endif + +/* + * 11. REDUCE CODE AND TABLE SIZE + * + * This replaces some expanded macros with function calls if AES_ASM_V2 or + * AES_ASM_V2C are defined + */ + +#if 1 && (defined(ASM_X86_V2) || defined(ASM_X86_V2C)) +#define REDUCE_CODE_SIZE +#endif + +/* + * 12. TABLE OPTIONS + * + * This cipher proceeds by repeating in a number of cycles known as rounds + * which are implemented by a round function which is optionally be speeded + * up using tables. The basic tables are 256 32-bit words, with either + * one or four tables being required for each round function depending on + * how much speed is required. Encryption and decryption round functions + * are different and the last encryption and decryption round functions are + * different again making four different round functions in all. + * + * This means that: + * 1. Normal encryption and decryption rounds can each use either 0, 1 + * or 4 tables and table spaces of 0, 1024 or 4096 bytes each. + * 2. The last encryption and decryption rounds can also use either 0, 1 + * or 4 tables and table spaces of 0, 1024 or 4096 bytes each. + * + * Include or exclude the appropriate definitions below to set the number + * of tables used by this implementation. + */ + +#if 1 /* set tables for the normal encryption round */ +#define ENC_ROUND FOUR_TABLES +#elif 0 +#define ENC_ROUND ONE_TABLE +#else +#define ENC_ROUND NO_TABLES +#endif + +#if 1 /* set tables for the last encryption round */ +#define LAST_ENC_ROUND FOUR_TABLES +#elif 0 +#define LAST_ENC_ROUND ONE_TABLE +#else +#define LAST_ENC_ROUND NO_TABLES +#endif + +#if 1 /* set tables for the normal decryption round */ +#define DEC_ROUND FOUR_TABLES +#elif 0 +#define DEC_ROUND ONE_TABLE +#else +#define DEC_ROUND NO_TABLES +#endif + +#if 1 /* set tables for the last decryption round */ +#define LAST_DEC_ROUND FOUR_TABLES +#elif 0 +#define LAST_DEC_ROUND ONE_TABLE +#else +#define LAST_DEC_ROUND NO_TABLES +#endif + +/* + * The decryption key schedule can be speeded up with tables in the same + * way that the round functions can. Include or exclude the following + * defines to set this requirement. + */ +#if 1 +#define KEY_SCHED FOUR_TABLES +#elif 0 +#define KEY_SCHED ONE_TABLE +#else +#define KEY_SCHED NO_TABLES +#endif + +/* ---- END OF USER CONFIGURED OPTIONS ---- */ + +/* VIA ACE support is only available for VC++ and GCC */ + +#if !defined(_MSC_VER) && !defined(__GNUC__) +#if defined(ASSUME_VIA_ACE_PRESENT) +#undef ASSUME_VIA_ACE_PRESENT +#endif +#if defined(USE_VIA_ACE_IF_PRESENT) +#undef USE_VIA_ACE_IF_PRESENT +#endif +#endif + +#if defined(ASSUME_VIA_ACE_PRESENT) && !defined(USE_VIA_ACE_IF_PRESENT) +#define USE_VIA_ACE_IF_PRESENT +#endif + +#if defined(USE_VIA_ACE_IF_PRESENT) && !defined(AES_REV_DKS) +#define AES_REV_DKS +#endif + +/* Assembler support requires the use of platform byte order */ + +#if (defined(ASM_X86_V1C) || defined(ASM_X86_V2C) || defined(ASM_AMD64_C)) && \ + (ALGORITHM_BYTE_ORDER != PLATFORM_BYTE_ORDER) +#undef ALGORITHM_BYTE_ORDER +#define ALGORITHM_BYTE_ORDER PLATFORM_BYTE_ORDER +#endif + +/* + * In this implementation the columns of the state array are each held in + * 32-bit words. The state array can be held in various ways: in an array + * of words, in a number of individual word variables or in a number of + * processor registers. The following define maps a variable name x and + * a column number c to the way the state array variable is to be held. + * The first define below maps the state into an array x[c] whereas the + * second form maps the state into a number of individual variables x0, + * x1, etc. Another form could map individual state columns to machine + * register names. + */ + +#if defined(ARRAYS) +#define s(x, c) x[c] +#else +#define s(x, c) x##c +#endif + +/* + * This implementation provides subroutines for encryption, decryption + * and for setting the three key lengths (separately) for encryption + * and decryption. Since not all functions are needed, masks are set + * up here to determine which will be implemented in C + */ + +#if !defined(AES_ENCRYPT) +#define EFUNCS_IN_C 0 +#elif defined(ASSUME_VIA_ACE_PRESENT) || defined(ASM_X86_V1C) || \ + defined(ASM_X86_V2C) || defined(ASM_AMD64_C) +#define EFUNCS_IN_C ENC_KEYING_IN_C +#elif !defined(ASM_X86_V2) +#define EFUNCS_IN_C (ENCRYPTION_IN_C | ENC_KEYING_IN_C) +#else +#define EFUNCS_IN_C 0 +#endif + +#if !defined(AES_DECRYPT) +#define DFUNCS_IN_C 0 +#elif defined(ASSUME_VIA_ACE_PRESENT) || defined(ASM_X86_V1C) || \ + defined(ASM_X86_V2C) || defined(ASM_AMD64_C) +#define DFUNCS_IN_C DEC_KEYING_IN_C +#elif !defined(ASM_X86_V2) +#define DFUNCS_IN_C (DECRYPTION_IN_C | DEC_KEYING_IN_C) +#else +#define DFUNCS_IN_C 0 +#endif + +#define FUNCS_IN_C (EFUNCS_IN_C | DFUNCS_IN_C) + +/* END OF CONFIGURATION OPTIONS */ + +/* Disable or report errors on some combinations of options */ + +#if ENC_ROUND == NO_TABLES && LAST_ENC_ROUND != NO_TABLES +#undef LAST_ENC_ROUND +#define LAST_ENC_ROUND NO_TABLES +#elif ENC_ROUND == ONE_TABLE && LAST_ENC_ROUND == FOUR_TABLES +#undef LAST_ENC_ROUND +#define LAST_ENC_ROUND ONE_TABLE +#endif + +#if ENC_ROUND == NO_TABLES && ENC_UNROLL != NONE +#undef ENC_UNROLL +#define ENC_UNROLL NONE +#endif + +#if DEC_ROUND == NO_TABLES && LAST_DEC_ROUND != NO_TABLES +#undef LAST_DEC_ROUND +#define LAST_DEC_ROUND NO_TABLES +#elif DEC_ROUND == ONE_TABLE && LAST_DEC_ROUND == FOUR_TABLES +#undef LAST_DEC_ROUND +#define LAST_DEC_ROUND ONE_TABLE +#endif + +#if DEC_ROUND == NO_TABLES && DEC_UNROLL != NONE +#undef DEC_UNROLL +#define DEC_UNROLL NONE +#endif + +#if (ALGORITHM_BYTE_ORDER == IS_LITTLE_ENDIAN) +#define aes_sw32 htonl +#elif defined(bswap32) +#define aes_sw32 bswap32 +#elif defined(bswap_32) +#define aes_sw32 bswap_32 +#else +#define brot(x, n) (((uint32_t)(x) << (n)) | ((uint32_t)(x) >> (32 - (n)))) +#define aes_sw32(x) ((brot((x), 8) & 0x00ff00ff) | (brot((x), 24) & 0xff00ff00)) +#endif + + +/* + * upr(x, n): rotates bytes within words by n positions, moving bytes to + * higher index positions with wrap around into low positions + * ups(x, n): moves bytes by n positions to higher index positions in + * words but without wrap around + * bval(x, n): extracts a byte from a word + * + * WARNING: The definitions given here are intended only for use with + * unsigned variables and with shift counts that are compile + * time constants + */ + +#if (ALGORITHM_BYTE_ORDER == IS_LITTLE_ENDIAN) +#define upr(x, n) (((uint32_t)(x) << (8 * (n))) | \ + ((uint32_t)(x) >> (32 - 8 * (n)))) +#define ups(x, n) ((uint32_t)(x) << (8 * (n))) +#define bval(x, n) to_byte((x) >> (8 * (n))) +#define bytes2word(b0, b1, b2, b3) \ + (((uint32_t)(b3) << 24) | ((uint32_t)(b2) << 16) | \ + ((uint32_t)(b1) << 8) | (b0)) +#endif + +#if (ALGORITHM_BYTE_ORDER == IS_BIG_ENDIAN) +#define upr(x, n) (((uint32_t)(x) >> (8 * (n))) | \ + ((uint32_t)(x) << (32 - 8 * (n)))) +#define ups(x, n) ((uint32_t)(x) >> (8 * (n))) +#define bval(x, n) to_byte((x) >> (24 - 8 * (n))) +#define bytes2word(b0, b1, b2, b3) \ + (((uint32_t)(b0) << 24) | ((uint32_t)(b1) << 16) | \ + ((uint32_t)(b2) << 8) | (b3)) +#endif + +#if defined(SAFE_IO) +#define word_in(x, c) bytes2word(((const uint8_t *)(x) + 4 * c)[0], \ + ((const uint8_t *)(x) + 4 * c)[1], \ + ((const uint8_t *)(x) + 4 * c)[2], \ + ((const uint8_t *)(x) + 4 * c)[3]) +#define word_out(x, c, v) { ((uint8_t *)(x) + 4 * c)[0] = bval(v, 0); \ + ((uint8_t *)(x) + 4 * c)[1] = bval(v, 1); \ + ((uint8_t *)(x) + 4 * c)[2] = bval(v, 2); \ + ((uint8_t *)(x) + 4 * c)[3] = bval(v, 3); } +#elif (ALGORITHM_BYTE_ORDER == PLATFORM_BYTE_ORDER) +#define word_in(x, c) (*((uint32_t *)(x) + (c))) +#define word_out(x, c, v) (*((uint32_t *)(x) + (c)) = (v)) +#else +#define word_in(x, c) aes_sw32(*((uint32_t *)(x) + (c))) +#define word_out(x, c, v) (*((uint32_t *)(x) + (c)) = aes_sw32(v)) +#endif + +/* the finite field modular polynomial and elements */ + +#define WPOLY 0x011b +#define BPOLY 0x1b + +/* multiply four bytes in GF(2^8) by 'x' {02} in parallel */ + +#define m1 0x80808080 +#define m2 0x7f7f7f7f +#define gf_mulx(x) ((((x) & m2) << 1) ^ ((((x) & m1) >> 7) * BPOLY)) + +/* + * The following defines provide alternative definitions of gf_mulx that might + * give improved performance if a fast 32-bit multiply is not available. Note + * that a temporary variable u needs to be defined where gf_mulx is used. + * + * #define gf_mulx(x) (u = (x) & m1, u |= (u >> 1), ((x) & m2) << 1) ^ \ + * ((u >> 3) | (u >> 6)) + * #define m4 (0x01010101 * BPOLY) + * #define gf_mulx(x) (u = (x) & m1, ((x) & m2) << 1) ^ ((u - (u >> 7)) \ + * & m4) + */ + +/* Work out which tables are needed for the different options */ + +#if defined(ASM_X86_V1C) +#if defined(ENC_ROUND) +#undef ENC_ROUND +#endif +#define ENC_ROUND FOUR_TABLES +#if defined(LAST_ENC_ROUND) +#undef LAST_ENC_ROUND +#endif +#define LAST_ENC_ROUND FOUR_TABLES +#if defined(DEC_ROUND) +#undef DEC_ROUND +#endif +#define DEC_ROUND FOUR_TABLES +#if defined(LAST_DEC_ROUND) +#undef LAST_DEC_ROUND +#endif +#define LAST_DEC_ROUND FOUR_TABLES +#if defined(KEY_SCHED) +#undef KEY_SCHED +#define KEY_SCHED FOUR_TABLES +#endif +#endif + +#if (FUNCS_IN_C & ENCRYPTION_IN_C) || defined(ASM_X86_V1C) +#if ENC_ROUND == ONE_TABLE +#define FT1_SET +#elif ENC_ROUND == FOUR_TABLES +#define FT4_SET +#else +#define SBX_SET +#endif +#if LAST_ENC_ROUND == ONE_TABLE +#define FL1_SET +#elif LAST_ENC_ROUND == FOUR_TABLES +#define FL4_SET +#elif !defined(SBX_SET) +#define SBX_SET +#endif +#endif + +#if (FUNCS_IN_C & DECRYPTION_IN_C) || defined(ASM_X86_V1C) +#if DEC_ROUND == ONE_TABLE +#define IT1_SET +#elif DEC_ROUND == FOUR_TABLES +#define IT4_SET +#else +#define ISB_SET +#endif +#if LAST_DEC_ROUND == ONE_TABLE +#define IL1_SET +#elif LAST_DEC_ROUND == FOUR_TABLES +#define IL4_SET +#elif !defined(ISB_SET) +#define ISB_SET +#endif +#endif + + +#if !(defined(REDUCE_CODE_SIZE) && (defined(ASM_X86_V2) || \ + defined(ASM_X86_V2C))) +#if ((FUNCS_IN_C & ENC_KEYING_IN_C) || (FUNCS_IN_C & DEC_KEYING_IN_C)) +#if KEY_SCHED == ONE_TABLE +#if !defined(FL1_SET) && !defined(FL4_SET) +#define LS1_SET +#endif +#elif KEY_SCHED == FOUR_TABLES +#if !defined(FL4_SET) +#define LS4_SET +#endif +#elif !defined(SBX_SET) +#define SBX_SET +#endif +#endif +#if (FUNCS_IN_C & DEC_KEYING_IN_C) +#if KEY_SCHED == ONE_TABLE +#define IM1_SET +#elif KEY_SCHED == FOUR_TABLES +#define IM4_SET +#elif !defined(SBX_SET) +#define SBX_SET +#endif +#endif +#endif + +/* generic definitions of Rijndael macros that use tables */ + +#define no_table(x, box, vf, rf, c) bytes2word(\ + box[bval(vf(x, 0, c), rf(0, c))], \ + box[bval(vf(x, 1, c), rf(1, c))], \ + box[bval(vf(x, 2, c), rf(2, c))], \ + box[bval(vf(x, 3, c), rf(3, c))]) + +#define one_table(x, op, tab, vf, rf, c) \ + (tab[bval(vf(x, 0, c), rf(0, c))] \ + ^ op(tab[bval(vf(x, 1, c), rf(1, c))], 1) \ + ^ op(tab[bval(vf(x, 2, c), rf(2, c))], 2) \ + ^ op(tab[bval(vf(x, 3, c), rf(3, c))], 3)) + +#define four_tables(x, tab, vf, rf, c) \ + (tab[0][bval(vf(x, 0, c), rf(0, c))] \ + ^ tab[1][bval(vf(x, 1, c), rf(1, c))] \ + ^ tab[2][bval(vf(x, 2, c), rf(2, c))] \ + ^ tab[3][bval(vf(x, 3, c), rf(3, c))]) + +#define vf1(x, r, c) (x) +#define rf1(r, c) (r) +#define rf2(r, c) ((8+r-c)&3) + +/* + * Perform forward and inverse column mix operation on four bytes in long word + * x in parallel. NOTE: x must be a simple variable, NOT an expression in + * these macros. + */ + +#if !(defined(REDUCE_CODE_SIZE) && (defined(ASM_X86_V2) || \ + defined(ASM_X86_V2C))) + +#if defined(FM4_SET) /* not currently used */ +#define fwd_mcol(x) four_tables(x, t_use(f, m), vf1, rf1, 0) +#elif defined(FM1_SET) /* not currently used */ +#define fwd_mcol(x) one_table(x, upr, t_use(f, m), vf1, rf1, 0) +#else +#define dec_fmvars uint32_t g2 +#define fwd_mcol(x) (g2 = gf_mulx(x), g2 ^ upr((x) ^ g2, 3) ^ \ + upr((x), 2) ^ upr((x), 1)) +#endif + +#if defined(IM4_SET) +#define inv_mcol(x) four_tables(x, t_use(i, m), vf1, rf1, 0) +#elif defined(IM1_SET) +#define inv_mcol(x) one_table(x, upr, t_use(i, m), vf1, rf1, 0) +#else +#define dec_imvars uint32_t g2, g4, g9 +#define inv_mcol(x) (g2 = gf_mulx(x), g4 = gf_mulx(g2), g9 = \ + (x) ^ gf_mulx(g4), g4 ^= g9, \ + (x) ^ g2 ^ g4 ^ upr(g2 ^ g9, 3) ^ \ + upr(g4, 2) ^ upr(g9, 1)) +#endif + +#if defined(FL4_SET) +#define ls_box(x, c) four_tables(x, t_use(f, l), vf1, rf2, c) +#elif defined(LS4_SET) +#define ls_box(x, c) four_tables(x, t_use(l, s), vf1, rf2, c) +#elif defined(FL1_SET) +#define ls_box(x, c) one_table(x, upr, t_use(f, l), vf1, rf2, c) +#elif defined(LS1_SET) +#define ls_box(x, c) one_table(x, upr, t_use(l, s), vf1, rf2, c) +#else +#define ls_box(x, c) no_table(x, t_use(s, box), vf1, rf2, c) +#endif + +#endif + +#if defined(ASM_X86_V1C) && defined(AES_DECRYPT) && !defined(ISB_SET) +#define ISB_SET +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _AESOPT_H */ diff --git a/module/icp/asm-x86_64/aes/aestab.h b/module/icp/asm-x86_64/aes/aestab.h new file mode 100644 index 000000000000..33cdb6c6f9fe --- /dev/null +++ b/module/icp/asm-x86_64/aes/aestab.h @@ -0,0 +1,165 @@ +/* + * --------------------------------------------------------------------------- + * Copyright (c) 1998-2007, Brian Gladman, Worcester, UK. All rights reserved. + * + * LICENSE TERMS + * + * The free distribution and use of this software is allowed (with or without + * changes) provided that: + * + * 1. source code distributions include the above copyright notice, this + * list of conditions and the following disclaimer; + * + * 2. binary distributions include the above copyright notice, this list + * of conditions and the following disclaimer in their documentation; + * + * 3. the name of the copyright holder is not used to endorse products + * built using this software without specific written permission. + * + * DISCLAIMER + * + * This software is provided 'as is' with no explicit or implied warranties + * in respect of its properties, including, but not limited to, correctness + * and/or fitness for purpose. + * --------------------------------------------------------------------------- + * Issue Date: 20/12/2007 + * + * This file contains the code for declaring the tables needed to implement + * AES. The file aesopt.h is assumed to be included before this header file. + * If there are no global variables, the definitions here can be used to put + * the AES tables in a structure so that a pointer can then be added to the + * AES context to pass them to the AES routines that need them. If this + * facility is used, the calling program has to ensure that this pointer is + * managed appropriately. In particular, the value of the t_dec(in, it) item + * in the table structure must be set to zero in order to ensure that the + * tables are initialised. In practice the three code sequences in aeskey.c + * that control the calls to aes_init() and the aes_init() routine itself will + * have to be changed for a specific implementation. If global variables are + * available it will generally be preferable to use them with the precomputed + * FIXED_TABLES option that uses static global tables. + * + * The following defines can be used to control the way the tables + * are defined, initialised and used in embedded environments that + * require special features for these purposes + * + * the 't_dec' construction is used to declare fixed table arrays + * the 't_set' construction is used to set fixed table values + * the 't_use' construction is used to access fixed table values + * + * 256 byte tables: + * + * t_xxx(s, box) => forward S box + * t_xxx(i, box) => inverse S box + * + * 256 32-bit word OR 4 x 256 32-bit word tables: + * + * t_xxx(f, n) => forward normal round + * t_xxx(f, l) => forward last round + * t_xxx(i, n) => inverse normal round + * t_xxx(i, l) => inverse last round + * t_xxx(l, s) => key schedule table + * t_xxx(i, m) => key schedule table + * + * Other variables and tables: + * + * t_xxx(r, c) => the rcon table + */ + +/* + * OpenSolaris OS modifications + * + * 1. Added __cplusplus and _AESTAB_H header guards + * 2. Added header file sys/types.h + * 3. Remove code defined for _MSC_VER + * 4. Changed all variables to "static const" + * 5. Changed uint_8t and uint_32t to uint8_t and uint32_t + * 6. Cstyled and hdrchk code + */ + +#ifndef _AESTAB_H +#define _AESTAB_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#define t_dec(m, n) t_##m##n +#define t_set(m, n) t_##m##n +#define t_use(m, n) t_##m##n + +#if defined(DO_TABLES) && defined(FIXED_TABLES) +#define d_1(t, n, b, e) static const t n[256] = b(e) +#define d_4(t, n, b, e, f, g, h) static const t n[4][256] = \ + {b(e), b(f), b(g), b(h)} +static const uint32_t t_dec(r, c)[RC_LENGTH] = rc_data(w0); +#else +#define d_1(t, n, b, e) static const t n[256] +#define d_4(t, n, b, e, f, g, h) static const t n[4][256] +static const uint32_t t_dec(r, c)[RC_LENGTH]; +#endif + +#if defined(SBX_SET) + d_1(uint8_t, t_dec(s, box), sb_data, h0); +#endif +#if defined(ISB_SET) + d_1(uint8_t, t_dec(i, box), isb_data, h0); +#endif + +#if defined(FT1_SET) + d_1(uint32_t, t_dec(f, n), sb_data, u0); +#endif +#if defined(FT4_SET) + d_4(uint32_t, t_dec(f, n), sb_data, u0, u1, u2, u3); +#endif + +#if defined(FL1_SET) + d_1(uint32_t, t_dec(f, l), sb_data, w0); +#endif +#if defined(FL4_SET) + d_4(uint32_t, t_dec(f, l), sb_data, w0, w1, w2, w3); +#endif + +#if defined(IT1_SET) + d_1(uint32_t, t_dec(i, n), isb_data, v0); +#endif +#if defined(IT4_SET) + d_4(uint32_t, t_dec(i, n), isb_data, v0, v1, v2, v3); +#endif + +#if defined(IL1_SET) + d_1(uint32_t, t_dec(i, l), isb_data, w0); +#endif +#if defined(IL4_SET) + d_4(uint32_t, t_dec(i, l), isb_data, w0, w1, w2, w3); +#endif + +#if defined(LS1_SET) +#if defined(FL1_SET) +#undef LS1_SET +#else + d_1(uint32_t, t_dec(l, s), sb_data, w0); +#endif +#endif + +#if defined(LS4_SET) +#if defined(FL4_SET) +#undef LS4_SET +#else + d_4(uint32_t, t_dec(l, s), sb_data, w0, w1, w2, w3); +#endif +#endif + +#if defined(IM1_SET) + d_1(uint32_t, t_dec(i, m), mm_data, v0); +#endif +#if defined(IM4_SET) + d_4(uint32_t, t_dec(i, m), mm_data, v0, v1, v2, v3); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* _AESTAB_H */ diff --git a/module/icp/asm-x86_64/aes/aestab2.h b/module/icp/asm-x86_64/aes/aestab2.h new file mode 100644 index 000000000000..eb13f72b10d8 --- /dev/null +++ b/module/icp/asm-x86_64/aes/aestab2.h @@ -0,0 +1,594 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _AESTAB2_H +#define _AESTAB2_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * To create this file for OpenSolaris: + * 1. Compile and run tablegen.c, from aes-src-04-03-08.zip, + * after defining ASM_AMD64_C + * 2. mv aestab2.c aestab2.h + * 3. Add __cplusplus and _AESTAB2_H header guards + * 3. Add #include + * 4. Change "uint_32t" to "uint32_t" + * 5. Change all variables to "static const" + * 6. Cstyle and hdrchk this file + */ + +#include + +static const uint32_t t_rc[RC_LENGTH] = +{ + 0x00000001, 0x00000002, 0x00000004, 0x00000008, + 0x00000010, 0x00000020, 0x00000040, 0x00000080, + 0x0000001b, 0x00000036 +}; + +static const uint32_t t_ls[4][256] = +{ + { + 0x00000063, 0x0000007c, 0x00000077, 0x0000007b, + 0x000000f2, 0x0000006b, 0x0000006f, 0x000000c5, + 0x00000030, 0x00000001, 0x00000067, 0x0000002b, + 0x000000fe, 0x000000d7, 0x000000ab, 0x00000076, + 0x000000ca, 0x00000082, 0x000000c9, 0x0000007d, + 0x000000fa, 0x00000059, 0x00000047, 0x000000f0, + 0x000000ad, 0x000000d4, 0x000000a2, 0x000000af, + 0x0000009c, 0x000000a4, 0x00000072, 0x000000c0, + 0x000000b7, 0x000000fd, 0x00000093, 0x00000026, + 0x00000036, 0x0000003f, 0x000000f7, 0x000000cc, + 0x00000034, 0x000000a5, 0x000000e5, 0x000000f1, + 0x00000071, 0x000000d8, 0x00000031, 0x00000015, + 0x00000004, 0x000000c7, 0x00000023, 0x000000c3, + 0x00000018, 0x00000096, 0x00000005, 0x0000009a, + 0x00000007, 0x00000012, 0x00000080, 0x000000e2, + 0x000000eb, 0x00000027, 0x000000b2, 0x00000075, + 0x00000009, 0x00000083, 0x0000002c, 0x0000001a, + 0x0000001b, 0x0000006e, 0x0000005a, 0x000000a0, + 0x00000052, 0x0000003b, 0x000000d6, 0x000000b3, + 0x00000029, 0x000000e3, 0x0000002f, 0x00000084, + 0x00000053, 0x000000d1, 0x00000000, 0x000000ed, + 0x00000020, 0x000000fc, 0x000000b1, 0x0000005b, + 0x0000006a, 0x000000cb, 0x000000be, 0x00000039, + 0x0000004a, 0x0000004c, 0x00000058, 0x000000cf, + 0x000000d0, 0x000000ef, 0x000000aa, 0x000000fb, + 0x00000043, 0x0000004d, 0x00000033, 0x00000085, + 0x00000045, 0x000000f9, 0x00000002, 0x0000007f, + 0x00000050, 0x0000003c, 0x0000009f, 0x000000a8, + 0x00000051, 0x000000a3, 0x00000040, 0x0000008f, + 0x00000092, 0x0000009d, 0x00000038, 0x000000f5, + 0x000000bc, 0x000000b6, 0x000000da, 0x00000021, + 0x00000010, 0x000000ff, 0x000000f3, 0x000000d2, + 0x000000cd, 0x0000000c, 0x00000013, 0x000000ec, + 0x0000005f, 0x00000097, 0x00000044, 0x00000017, + 0x000000c4, 0x000000a7, 0x0000007e, 0x0000003d, + 0x00000064, 0x0000005d, 0x00000019, 0x00000073, + 0x00000060, 0x00000081, 0x0000004f, 0x000000dc, + 0x00000022, 0x0000002a, 0x00000090, 0x00000088, + 0x00000046, 0x000000ee, 0x000000b8, 0x00000014, + 0x000000de, 0x0000005e, 0x0000000b, 0x000000db, + 0x000000e0, 0x00000032, 0x0000003a, 0x0000000a, + 0x00000049, 0x00000006, 0x00000024, 0x0000005c, + 0x000000c2, 0x000000d3, 0x000000ac, 0x00000062, + 0x00000091, 0x00000095, 0x000000e4, 0x00000079, + 0x000000e7, 0x000000c8, 0x00000037, 0x0000006d, + 0x0000008d, 0x000000d5, 0x0000004e, 0x000000a9, + 0x0000006c, 0x00000056, 0x000000f4, 0x000000ea, + 0x00000065, 0x0000007a, 0x000000ae, 0x00000008, + 0x000000ba, 0x00000078, 0x00000025, 0x0000002e, + 0x0000001c, 0x000000a6, 0x000000b4, 0x000000c6, + 0x000000e8, 0x000000dd, 0x00000074, 0x0000001f, + 0x0000004b, 0x000000bd, 0x0000008b, 0x0000008a, + 0x00000070, 0x0000003e, 0x000000b5, 0x00000066, + 0x00000048, 0x00000003, 0x000000f6, 0x0000000e, + 0x00000061, 0x00000035, 0x00000057, 0x000000b9, + 0x00000086, 0x000000c1, 0x0000001d, 0x0000009e, + 0x000000e1, 0x000000f8, 0x00000098, 0x00000011, + 0x00000069, 0x000000d9, 0x0000008e, 0x00000094, + 0x0000009b, 0x0000001e, 0x00000087, 0x000000e9, + 0x000000ce, 0x00000055, 0x00000028, 0x000000df, + 0x0000008c, 0x000000a1, 0x00000089, 0x0000000d, + 0x000000bf, 0x000000e6, 0x00000042, 0x00000068, + 0x00000041, 0x00000099, 0x0000002d, 0x0000000f, + 0x000000b0, 0x00000054, 0x000000bb, 0x00000016 + }, + { + 0x00006300, 0x00007c00, 0x00007700, 0x00007b00, + 0x0000f200, 0x00006b00, 0x00006f00, 0x0000c500, + 0x00003000, 0x00000100, 0x00006700, 0x00002b00, + 0x0000fe00, 0x0000d700, 0x0000ab00, 0x00007600, + 0x0000ca00, 0x00008200, 0x0000c900, 0x00007d00, + 0x0000fa00, 0x00005900, 0x00004700, 0x0000f000, + 0x0000ad00, 0x0000d400, 0x0000a200, 0x0000af00, + 0x00009c00, 0x0000a400, 0x00007200, 0x0000c000, + 0x0000b700, 0x0000fd00, 0x00009300, 0x00002600, + 0x00003600, 0x00003f00, 0x0000f700, 0x0000cc00, + 0x00003400, 0x0000a500, 0x0000e500, 0x0000f100, + 0x00007100, 0x0000d800, 0x00003100, 0x00001500, + 0x00000400, 0x0000c700, 0x00002300, 0x0000c300, + 0x00001800, 0x00009600, 0x00000500, 0x00009a00, + 0x00000700, 0x00001200, 0x00008000, 0x0000e200, + 0x0000eb00, 0x00002700, 0x0000b200, 0x00007500, + 0x00000900, 0x00008300, 0x00002c00, 0x00001a00, + 0x00001b00, 0x00006e00, 0x00005a00, 0x0000a000, + 0x00005200, 0x00003b00, 0x0000d600, 0x0000b300, + 0x00002900, 0x0000e300, 0x00002f00, 0x00008400, + 0x00005300, 0x0000d100, 0x00000000, 0x0000ed00, + 0x00002000, 0x0000fc00, 0x0000b100, 0x00005b00, + 0x00006a00, 0x0000cb00, 0x0000be00, 0x00003900, + 0x00004a00, 0x00004c00, 0x00005800, 0x0000cf00, + 0x0000d000, 0x0000ef00, 0x0000aa00, 0x0000fb00, + 0x00004300, 0x00004d00, 0x00003300, 0x00008500, + 0x00004500, 0x0000f900, 0x00000200, 0x00007f00, + 0x00005000, 0x00003c00, 0x00009f00, 0x0000a800, + 0x00005100, 0x0000a300, 0x00004000, 0x00008f00, + 0x00009200, 0x00009d00, 0x00003800, 0x0000f500, + 0x0000bc00, 0x0000b600, 0x0000da00, 0x00002100, + 0x00001000, 0x0000ff00, 0x0000f300, 0x0000d200, + 0x0000cd00, 0x00000c00, 0x00001300, 0x0000ec00, + 0x00005f00, 0x00009700, 0x00004400, 0x00001700, + 0x0000c400, 0x0000a700, 0x00007e00, 0x00003d00, + 0x00006400, 0x00005d00, 0x00001900, 0x00007300, + 0x00006000, 0x00008100, 0x00004f00, 0x0000dc00, + 0x00002200, 0x00002a00, 0x00009000, 0x00008800, + 0x00004600, 0x0000ee00, 0x0000b800, 0x00001400, + 0x0000de00, 0x00005e00, 0x00000b00, 0x0000db00, + 0x0000e000, 0x00003200, 0x00003a00, 0x00000a00, + 0x00004900, 0x00000600, 0x00002400, 0x00005c00, + 0x0000c200, 0x0000d300, 0x0000ac00, 0x00006200, + 0x00009100, 0x00009500, 0x0000e400, 0x00007900, + 0x0000e700, 0x0000c800, 0x00003700, 0x00006d00, + 0x00008d00, 0x0000d500, 0x00004e00, 0x0000a900, + 0x00006c00, 0x00005600, 0x0000f400, 0x0000ea00, + 0x00006500, 0x00007a00, 0x0000ae00, 0x00000800, + 0x0000ba00, 0x00007800, 0x00002500, 0x00002e00, + 0x00001c00, 0x0000a600, 0x0000b400, 0x0000c600, + 0x0000e800, 0x0000dd00, 0x00007400, 0x00001f00, + 0x00004b00, 0x0000bd00, 0x00008b00, 0x00008a00, + 0x00007000, 0x00003e00, 0x0000b500, 0x00006600, + 0x00004800, 0x00000300, 0x0000f600, 0x00000e00, + 0x00006100, 0x00003500, 0x00005700, 0x0000b900, + 0x00008600, 0x0000c100, 0x00001d00, 0x00009e00, + 0x0000e100, 0x0000f800, 0x00009800, 0x00001100, + 0x00006900, 0x0000d900, 0x00008e00, 0x00009400, + 0x00009b00, 0x00001e00, 0x00008700, 0x0000e900, + 0x0000ce00, 0x00005500, 0x00002800, 0x0000df00, + 0x00008c00, 0x0000a100, 0x00008900, 0x00000d00, + 0x0000bf00, 0x0000e600, 0x00004200, 0x00006800, + 0x00004100, 0x00009900, 0x00002d00, 0x00000f00, + 0x0000b000, 0x00005400, 0x0000bb00, 0x00001600 + }, + { + 0x00630000, 0x007c0000, 0x00770000, 0x007b0000, + 0x00f20000, 0x006b0000, 0x006f0000, 0x00c50000, + 0x00300000, 0x00010000, 0x00670000, 0x002b0000, + 0x00fe0000, 0x00d70000, 0x00ab0000, 0x00760000, + 0x00ca0000, 0x00820000, 0x00c90000, 0x007d0000, + 0x00fa0000, 0x00590000, 0x00470000, 0x00f00000, + 0x00ad0000, 0x00d40000, 0x00a20000, 0x00af0000, + 0x009c0000, 0x00a40000, 0x00720000, 0x00c00000, + 0x00b70000, 0x00fd0000, 0x00930000, 0x00260000, + 0x00360000, 0x003f0000, 0x00f70000, 0x00cc0000, + 0x00340000, 0x00a50000, 0x00e50000, 0x00f10000, + 0x00710000, 0x00d80000, 0x00310000, 0x00150000, + 0x00040000, 0x00c70000, 0x00230000, 0x00c30000, + 0x00180000, 0x00960000, 0x00050000, 0x009a0000, + 0x00070000, 0x00120000, 0x00800000, 0x00e20000, + 0x00eb0000, 0x00270000, 0x00b20000, 0x00750000, + 0x00090000, 0x00830000, 0x002c0000, 0x001a0000, + 0x001b0000, 0x006e0000, 0x005a0000, 0x00a00000, + 0x00520000, 0x003b0000, 0x00d60000, 0x00b30000, + 0x00290000, 0x00e30000, 0x002f0000, 0x00840000, + 0x00530000, 0x00d10000, 0x00000000, 0x00ed0000, + 0x00200000, 0x00fc0000, 0x00b10000, 0x005b0000, + 0x006a0000, 0x00cb0000, 0x00be0000, 0x00390000, + 0x004a0000, 0x004c0000, 0x00580000, 0x00cf0000, + 0x00d00000, 0x00ef0000, 0x00aa0000, 0x00fb0000, + 0x00430000, 0x004d0000, 0x00330000, 0x00850000, + 0x00450000, 0x00f90000, 0x00020000, 0x007f0000, + 0x00500000, 0x003c0000, 0x009f0000, 0x00a80000, + 0x00510000, 0x00a30000, 0x00400000, 0x008f0000, + 0x00920000, 0x009d0000, 0x00380000, 0x00f50000, + 0x00bc0000, 0x00b60000, 0x00da0000, 0x00210000, + 0x00100000, 0x00ff0000, 0x00f30000, 0x00d20000, + 0x00cd0000, 0x000c0000, 0x00130000, 0x00ec0000, + 0x005f0000, 0x00970000, 0x00440000, 0x00170000, + 0x00c40000, 0x00a70000, 0x007e0000, 0x003d0000, + 0x00640000, 0x005d0000, 0x00190000, 0x00730000, + 0x00600000, 0x00810000, 0x004f0000, 0x00dc0000, + 0x00220000, 0x002a0000, 0x00900000, 0x00880000, + 0x00460000, 0x00ee0000, 0x00b80000, 0x00140000, + 0x00de0000, 0x005e0000, 0x000b0000, 0x00db0000, + 0x00e00000, 0x00320000, 0x003a0000, 0x000a0000, + 0x00490000, 0x00060000, 0x00240000, 0x005c0000, + 0x00c20000, 0x00d30000, 0x00ac0000, 0x00620000, + 0x00910000, 0x00950000, 0x00e40000, 0x00790000, + 0x00e70000, 0x00c80000, 0x00370000, 0x006d0000, + 0x008d0000, 0x00d50000, 0x004e0000, 0x00a90000, + 0x006c0000, 0x00560000, 0x00f40000, 0x00ea0000, + 0x00650000, 0x007a0000, 0x00ae0000, 0x00080000, + 0x00ba0000, 0x00780000, 0x00250000, 0x002e0000, + 0x001c0000, 0x00a60000, 0x00b40000, 0x00c60000, + 0x00e80000, 0x00dd0000, 0x00740000, 0x001f0000, + 0x004b0000, 0x00bd0000, 0x008b0000, 0x008a0000, + 0x00700000, 0x003e0000, 0x00b50000, 0x00660000, + 0x00480000, 0x00030000, 0x00f60000, 0x000e0000, + 0x00610000, 0x00350000, 0x00570000, 0x00b90000, + 0x00860000, 0x00c10000, 0x001d0000, 0x009e0000, + 0x00e10000, 0x00f80000, 0x00980000, 0x00110000, + 0x00690000, 0x00d90000, 0x008e0000, 0x00940000, + 0x009b0000, 0x001e0000, 0x00870000, 0x00e90000, + 0x00ce0000, 0x00550000, 0x00280000, 0x00df0000, + 0x008c0000, 0x00a10000, 0x00890000, 0x000d0000, + 0x00bf0000, 0x00e60000, 0x00420000, 0x00680000, + 0x00410000, 0x00990000, 0x002d0000, 0x000f0000, + 0x00b00000, 0x00540000, 0x00bb0000, 0x00160000 + }, + { + 0x63000000, 0x7c000000, 0x77000000, 0x7b000000, + 0xf2000000, 0x6b000000, 0x6f000000, 0xc5000000, + 0x30000000, 0x01000000, 0x67000000, 0x2b000000, + 0xfe000000, 0xd7000000, 0xab000000, 0x76000000, + 0xca000000, 0x82000000, 0xc9000000, 0x7d000000, + 0xfa000000, 0x59000000, 0x47000000, 0xf0000000, + 0xad000000, 0xd4000000, 0xa2000000, 0xaf000000, + 0x9c000000, 0xa4000000, 0x72000000, 0xc0000000, + 0xb7000000, 0xfd000000, 0x93000000, 0x26000000, + 0x36000000, 0x3f000000, 0xf7000000, 0xcc000000, + 0x34000000, 0xa5000000, 0xe5000000, 0xf1000000, + 0x71000000, 0xd8000000, 0x31000000, 0x15000000, + 0x04000000, 0xc7000000, 0x23000000, 0xc3000000, + 0x18000000, 0x96000000, 0x05000000, 0x9a000000, + 0x07000000, 0x12000000, 0x80000000, 0xe2000000, + 0xeb000000, 0x27000000, 0xb2000000, 0x75000000, + 0x09000000, 0x83000000, 0x2c000000, 0x1a000000, + 0x1b000000, 0x6e000000, 0x5a000000, 0xa0000000, + 0x52000000, 0x3b000000, 0xd6000000, 0xb3000000, + 0x29000000, 0xe3000000, 0x2f000000, 0x84000000, + 0x53000000, 0xd1000000, 0x00000000, 0xed000000, + 0x20000000, 0xfc000000, 0xb1000000, 0x5b000000, + 0x6a000000, 0xcb000000, 0xbe000000, 0x39000000, + 0x4a000000, 0x4c000000, 0x58000000, 0xcf000000, + 0xd0000000, 0xef000000, 0xaa000000, 0xfb000000, + 0x43000000, 0x4d000000, 0x33000000, 0x85000000, + 0x45000000, 0xf9000000, 0x02000000, 0x7f000000, + 0x50000000, 0x3c000000, 0x9f000000, 0xa8000000, + 0x51000000, 0xa3000000, 0x40000000, 0x8f000000, + 0x92000000, 0x9d000000, 0x38000000, 0xf5000000, + 0xbc000000, 0xb6000000, 0xda000000, 0x21000000, + 0x10000000, 0xff000000, 0xf3000000, 0xd2000000, + 0xcd000000, 0x0c000000, 0x13000000, 0xec000000, + 0x5f000000, 0x97000000, 0x44000000, 0x17000000, + 0xc4000000, 0xa7000000, 0x7e000000, 0x3d000000, + 0x64000000, 0x5d000000, 0x19000000, 0x73000000, + 0x60000000, 0x81000000, 0x4f000000, 0xdc000000, + 0x22000000, 0x2a000000, 0x90000000, 0x88000000, + 0x46000000, 0xee000000, 0xb8000000, 0x14000000, + 0xde000000, 0x5e000000, 0x0b000000, 0xdb000000, + 0xe0000000, 0x32000000, 0x3a000000, 0x0a000000, + 0x49000000, 0x06000000, 0x24000000, 0x5c000000, + 0xc2000000, 0xd3000000, 0xac000000, 0x62000000, + 0x91000000, 0x95000000, 0xe4000000, 0x79000000, + 0xe7000000, 0xc8000000, 0x37000000, 0x6d000000, + 0x8d000000, 0xd5000000, 0x4e000000, 0xa9000000, + 0x6c000000, 0x56000000, 0xf4000000, 0xea000000, + 0x65000000, 0x7a000000, 0xae000000, 0x08000000, + 0xba000000, 0x78000000, 0x25000000, 0x2e000000, + 0x1c000000, 0xa6000000, 0xb4000000, 0xc6000000, + 0xe8000000, 0xdd000000, 0x74000000, 0x1f000000, + 0x4b000000, 0xbd000000, 0x8b000000, 0x8a000000, + 0x70000000, 0x3e000000, 0xb5000000, 0x66000000, + 0x48000000, 0x03000000, 0xf6000000, 0x0e000000, + 0x61000000, 0x35000000, 0x57000000, 0xb9000000, + 0x86000000, 0xc1000000, 0x1d000000, 0x9e000000, + 0xe1000000, 0xf8000000, 0x98000000, 0x11000000, + 0x69000000, 0xd9000000, 0x8e000000, 0x94000000, + 0x9b000000, 0x1e000000, 0x87000000, 0xe9000000, + 0xce000000, 0x55000000, 0x28000000, 0xdf000000, + 0x8c000000, 0xa1000000, 0x89000000, 0x0d000000, + 0xbf000000, 0xe6000000, 0x42000000, 0x68000000, + 0x41000000, 0x99000000, 0x2d000000, 0x0f000000, + 0xb0000000, 0x54000000, 0xbb000000, 0x16000000 + } +}; + +static const uint32_t t_im[4][256] = +{ + { + 0x00000000, 0x0b0d090e, 0x161a121c, 0x1d171b12, + 0x2c342438, 0x27392d36, 0x3a2e3624, 0x31233f2a, + 0x58684870, 0x5365417e, 0x4e725a6c, 0x457f5362, + 0x745c6c48, 0x7f516546, 0x62467e54, 0x694b775a, + 0xb0d090e0, 0xbbdd99ee, 0xa6ca82fc, 0xadc78bf2, + 0x9ce4b4d8, 0x97e9bdd6, 0x8afea6c4, 0x81f3afca, + 0xe8b8d890, 0xe3b5d19e, 0xfea2ca8c, 0xf5afc382, + 0xc48cfca8, 0xcf81f5a6, 0xd296eeb4, 0xd99be7ba, + 0x7bbb3bdb, 0x70b632d5, 0x6da129c7, 0x66ac20c9, + 0x578f1fe3, 0x5c8216ed, 0x41950dff, 0x4a9804f1, + 0x23d373ab, 0x28de7aa5, 0x35c961b7, 0x3ec468b9, + 0x0fe75793, 0x04ea5e9d, 0x19fd458f, 0x12f04c81, + 0xcb6bab3b, 0xc066a235, 0xdd71b927, 0xd67cb029, + 0xe75f8f03, 0xec52860d, 0xf1459d1f, 0xfa489411, + 0x9303e34b, 0x980eea45, 0x8519f157, 0x8e14f859, + 0xbf37c773, 0xb43ace7d, 0xa92dd56f, 0xa220dc61, + 0xf66d76ad, 0xfd607fa3, 0xe07764b1, 0xeb7a6dbf, + 0xda595295, 0xd1545b9b, 0xcc434089, 0xc74e4987, + 0xae053edd, 0xa50837d3, 0xb81f2cc1, 0xb31225cf, + 0x82311ae5, 0x893c13eb, 0x942b08f9, 0x9f2601f7, + 0x46bde64d, 0x4db0ef43, 0x50a7f451, 0x5baafd5f, + 0x6a89c275, 0x6184cb7b, 0x7c93d069, 0x779ed967, + 0x1ed5ae3d, 0x15d8a733, 0x08cfbc21, 0x03c2b52f, + 0x32e18a05, 0x39ec830b, 0x24fb9819, 0x2ff69117, + 0x8dd64d76, 0x86db4478, 0x9bcc5f6a, 0x90c15664, + 0xa1e2694e, 0xaaef6040, 0xb7f87b52, 0xbcf5725c, + 0xd5be0506, 0xdeb30c08, 0xc3a4171a, 0xc8a91e14, + 0xf98a213e, 0xf2872830, 0xef903322, 0xe49d3a2c, + 0x3d06dd96, 0x360bd498, 0x2b1ccf8a, 0x2011c684, + 0x1132f9ae, 0x1a3ff0a0, 0x0728ebb2, 0x0c25e2bc, + 0x656e95e6, 0x6e639ce8, 0x737487fa, 0x78798ef4, + 0x495ab1de, 0x4257b8d0, 0x5f40a3c2, 0x544daacc, + 0xf7daec41, 0xfcd7e54f, 0xe1c0fe5d, 0xeacdf753, + 0xdbeec879, 0xd0e3c177, 0xcdf4da65, 0xc6f9d36b, + 0xafb2a431, 0xa4bfad3f, 0xb9a8b62d, 0xb2a5bf23, + 0x83868009, 0x888b8907, 0x959c9215, 0x9e919b1b, + 0x470a7ca1, 0x4c0775af, 0x51106ebd, 0x5a1d67b3, + 0x6b3e5899, 0x60335197, 0x7d244a85, 0x7629438b, + 0x1f6234d1, 0x146f3ddf, 0x097826cd, 0x02752fc3, + 0x335610e9, 0x385b19e7, 0x254c02f5, 0x2e410bfb, + 0x8c61d79a, 0x876cde94, 0x9a7bc586, 0x9176cc88, + 0xa055f3a2, 0xab58faac, 0xb64fe1be, 0xbd42e8b0, + 0xd4099fea, 0xdf0496e4, 0xc2138df6, 0xc91e84f8, + 0xf83dbbd2, 0xf330b2dc, 0xee27a9ce, 0xe52aa0c0, + 0x3cb1477a, 0x37bc4e74, 0x2aab5566, 0x21a65c68, + 0x10856342, 0x1b886a4c, 0x069f715e, 0x0d927850, + 0x64d90f0a, 0x6fd40604, 0x72c31d16, 0x79ce1418, + 0x48ed2b32, 0x43e0223c, 0x5ef7392e, 0x55fa3020, + 0x01b79aec, 0x0aba93e2, 0x17ad88f0, 0x1ca081fe, + 0x2d83bed4, 0x268eb7da, 0x3b99acc8, 0x3094a5c6, + 0x59dfd29c, 0x52d2db92, 0x4fc5c080, 0x44c8c98e, + 0x75ebf6a4, 0x7ee6ffaa, 0x63f1e4b8, 0x68fcedb6, + 0xb1670a0c, 0xba6a0302, 0xa77d1810, 0xac70111e, + 0x9d532e34, 0x965e273a, 0x8b493c28, 0x80443526, + 0xe90f427c, 0xe2024b72, 0xff155060, 0xf418596e, + 0xc53b6644, 0xce366f4a, 0xd3217458, 0xd82c7d56, + 0x7a0ca137, 0x7101a839, 0x6c16b32b, 0x671bba25, + 0x5638850f, 0x5d358c01, 0x40229713, 0x4b2f9e1d, + 0x2264e947, 0x2969e049, 0x347efb5b, 0x3f73f255, + 0x0e50cd7f, 0x055dc471, 0x184adf63, 0x1347d66d, + 0xcadc31d7, 0xc1d138d9, 0xdcc623cb, 0xd7cb2ac5, + 0xe6e815ef, 0xede51ce1, 0xf0f207f3, 0xfbff0efd, + 0x92b479a7, 0x99b970a9, 0x84ae6bbb, 0x8fa362b5, + 0xbe805d9f, 0xb58d5491, 0xa89a4f83, 0xa397468d + }, + { + 0x00000000, 0x0d090e0b, 0x1a121c16, 0x171b121d, + 0x3424382c, 0x392d3627, 0x2e36243a, 0x233f2a31, + 0x68487058, 0x65417e53, 0x725a6c4e, 0x7f536245, + 0x5c6c4874, 0x5165467f, 0x467e5462, 0x4b775a69, + 0xd090e0b0, 0xdd99eebb, 0xca82fca6, 0xc78bf2ad, + 0xe4b4d89c, 0xe9bdd697, 0xfea6c48a, 0xf3afca81, + 0xb8d890e8, 0xb5d19ee3, 0xa2ca8cfe, 0xafc382f5, + 0x8cfca8c4, 0x81f5a6cf, 0x96eeb4d2, 0x9be7bad9, + 0xbb3bdb7b, 0xb632d570, 0xa129c76d, 0xac20c966, + 0x8f1fe357, 0x8216ed5c, 0x950dff41, 0x9804f14a, + 0xd373ab23, 0xde7aa528, 0xc961b735, 0xc468b93e, + 0xe757930f, 0xea5e9d04, 0xfd458f19, 0xf04c8112, + 0x6bab3bcb, 0x66a235c0, 0x71b927dd, 0x7cb029d6, + 0x5f8f03e7, 0x52860dec, 0x459d1ff1, 0x489411fa, + 0x03e34b93, 0x0eea4598, 0x19f15785, 0x14f8598e, + 0x37c773bf, 0x3ace7db4, 0x2dd56fa9, 0x20dc61a2, + 0x6d76adf6, 0x607fa3fd, 0x7764b1e0, 0x7a6dbfeb, + 0x595295da, 0x545b9bd1, 0x434089cc, 0x4e4987c7, + 0x053eddae, 0x0837d3a5, 0x1f2cc1b8, 0x1225cfb3, + 0x311ae582, 0x3c13eb89, 0x2b08f994, 0x2601f79f, + 0xbde64d46, 0xb0ef434d, 0xa7f45150, 0xaafd5f5b, + 0x89c2756a, 0x84cb7b61, 0x93d0697c, 0x9ed96777, + 0xd5ae3d1e, 0xd8a73315, 0xcfbc2108, 0xc2b52f03, + 0xe18a0532, 0xec830b39, 0xfb981924, 0xf691172f, + 0xd64d768d, 0xdb447886, 0xcc5f6a9b, 0xc1566490, + 0xe2694ea1, 0xef6040aa, 0xf87b52b7, 0xf5725cbc, + 0xbe0506d5, 0xb30c08de, 0xa4171ac3, 0xa91e14c8, + 0x8a213ef9, 0x872830f2, 0x903322ef, 0x9d3a2ce4, + 0x06dd963d, 0x0bd49836, 0x1ccf8a2b, 0x11c68420, + 0x32f9ae11, 0x3ff0a01a, 0x28ebb207, 0x25e2bc0c, + 0x6e95e665, 0x639ce86e, 0x7487fa73, 0x798ef478, + 0x5ab1de49, 0x57b8d042, 0x40a3c25f, 0x4daacc54, + 0xdaec41f7, 0xd7e54ffc, 0xc0fe5de1, 0xcdf753ea, + 0xeec879db, 0xe3c177d0, 0xf4da65cd, 0xf9d36bc6, + 0xb2a431af, 0xbfad3fa4, 0xa8b62db9, 0xa5bf23b2, + 0x86800983, 0x8b890788, 0x9c921595, 0x919b1b9e, + 0x0a7ca147, 0x0775af4c, 0x106ebd51, 0x1d67b35a, + 0x3e58996b, 0x33519760, 0x244a857d, 0x29438b76, + 0x6234d11f, 0x6f3ddf14, 0x7826cd09, 0x752fc302, + 0x5610e933, 0x5b19e738, 0x4c02f525, 0x410bfb2e, + 0x61d79a8c, 0x6cde9487, 0x7bc5869a, 0x76cc8891, + 0x55f3a2a0, 0x58faacab, 0x4fe1beb6, 0x42e8b0bd, + 0x099fead4, 0x0496e4df, 0x138df6c2, 0x1e84f8c9, + 0x3dbbd2f8, 0x30b2dcf3, 0x27a9ceee, 0x2aa0c0e5, + 0xb1477a3c, 0xbc4e7437, 0xab55662a, 0xa65c6821, + 0x85634210, 0x886a4c1b, 0x9f715e06, 0x9278500d, + 0xd90f0a64, 0xd406046f, 0xc31d1672, 0xce141879, + 0xed2b3248, 0xe0223c43, 0xf7392e5e, 0xfa302055, + 0xb79aec01, 0xba93e20a, 0xad88f017, 0xa081fe1c, + 0x83bed42d, 0x8eb7da26, 0x99acc83b, 0x94a5c630, + 0xdfd29c59, 0xd2db9252, 0xc5c0804f, 0xc8c98e44, + 0xebf6a475, 0xe6ffaa7e, 0xf1e4b863, 0xfcedb668, + 0x670a0cb1, 0x6a0302ba, 0x7d1810a7, 0x70111eac, + 0x532e349d, 0x5e273a96, 0x493c288b, 0x44352680, + 0x0f427ce9, 0x024b72e2, 0x155060ff, 0x18596ef4, + 0x3b6644c5, 0x366f4ace, 0x217458d3, 0x2c7d56d8, + 0x0ca1377a, 0x01a83971, 0x16b32b6c, 0x1bba2567, + 0x38850f56, 0x358c015d, 0x22971340, 0x2f9e1d4b, + 0x64e94722, 0x69e04929, 0x7efb5b34, 0x73f2553f, + 0x50cd7f0e, 0x5dc47105, 0x4adf6318, 0x47d66d13, + 0xdc31d7ca, 0xd138d9c1, 0xc623cbdc, 0xcb2ac5d7, + 0xe815efe6, 0xe51ce1ed, 0xf207f3f0, 0xff0efdfb, + 0xb479a792, 0xb970a999, 0xae6bbb84, 0xa362b58f, + 0x805d9fbe, 0x8d5491b5, 0x9a4f83a8, 0x97468da3 + }, + { + 0x00000000, 0x090e0b0d, 0x121c161a, 0x1b121d17, + 0x24382c34, 0x2d362739, 0x36243a2e, 0x3f2a3123, + 0x48705868, 0x417e5365, 0x5a6c4e72, 0x5362457f, + 0x6c48745c, 0x65467f51, 0x7e546246, 0x775a694b, + 0x90e0b0d0, 0x99eebbdd, 0x82fca6ca, 0x8bf2adc7, + 0xb4d89ce4, 0xbdd697e9, 0xa6c48afe, 0xafca81f3, + 0xd890e8b8, 0xd19ee3b5, 0xca8cfea2, 0xc382f5af, + 0xfca8c48c, 0xf5a6cf81, 0xeeb4d296, 0xe7bad99b, + 0x3bdb7bbb, 0x32d570b6, 0x29c76da1, 0x20c966ac, + 0x1fe3578f, 0x16ed5c82, 0x0dff4195, 0x04f14a98, + 0x73ab23d3, 0x7aa528de, 0x61b735c9, 0x68b93ec4, + 0x57930fe7, 0x5e9d04ea, 0x458f19fd, 0x4c8112f0, + 0xab3bcb6b, 0xa235c066, 0xb927dd71, 0xb029d67c, + 0x8f03e75f, 0x860dec52, 0x9d1ff145, 0x9411fa48, + 0xe34b9303, 0xea45980e, 0xf1578519, 0xf8598e14, + 0xc773bf37, 0xce7db43a, 0xd56fa92d, 0xdc61a220, + 0x76adf66d, 0x7fa3fd60, 0x64b1e077, 0x6dbfeb7a, + 0x5295da59, 0x5b9bd154, 0x4089cc43, 0x4987c74e, + 0x3eddae05, 0x37d3a508, 0x2cc1b81f, 0x25cfb312, + 0x1ae58231, 0x13eb893c, 0x08f9942b, 0x01f79f26, + 0xe64d46bd, 0xef434db0, 0xf45150a7, 0xfd5f5baa, + 0xc2756a89, 0xcb7b6184, 0xd0697c93, 0xd967779e, + 0xae3d1ed5, 0xa73315d8, 0xbc2108cf, 0xb52f03c2, + 0x8a0532e1, 0x830b39ec, 0x981924fb, 0x91172ff6, + 0x4d768dd6, 0x447886db, 0x5f6a9bcc, 0x566490c1, + 0x694ea1e2, 0x6040aaef, 0x7b52b7f8, 0x725cbcf5, + 0x0506d5be, 0x0c08deb3, 0x171ac3a4, 0x1e14c8a9, + 0x213ef98a, 0x2830f287, 0x3322ef90, 0x3a2ce49d, + 0xdd963d06, 0xd498360b, 0xcf8a2b1c, 0xc6842011, + 0xf9ae1132, 0xf0a01a3f, 0xebb20728, 0xe2bc0c25, + 0x95e6656e, 0x9ce86e63, 0x87fa7374, 0x8ef47879, + 0xb1de495a, 0xb8d04257, 0xa3c25f40, 0xaacc544d, + 0xec41f7da, 0xe54ffcd7, 0xfe5de1c0, 0xf753eacd, + 0xc879dbee, 0xc177d0e3, 0xda65cdf4, 0xd36bc6f9, + 0xa431afb2, 0xad3fa4bf, 0xb62db9a8, 0xbf23b2a5, + 0x80098386, 0x8907888b, 0x9215959c, 0x9b1b9e91, + 0x7ca1470a, 0x75af4c07, 0x6ebd5110, 0x67b35a1d, + 0x58996b3e, 0x51976033, 0x4a857d24, 0x438b7629, + 0x34d11f62, 0x3ddf146f, 0x26cd0978, 0x2fc30275, + 0x10e93356, 0x19e7385b, 0x02f5254c, 0x0bfb2e41, + 0xd79a8c61, 0xde94876c, 0xc5869a7b, 0xcc889176, + 0xf3a2a055, 0xfaacab58, 0xe1beb64f, 0xe8b0bd42, + 0x9fead409, 0x96e4df04, 0x8df6c213, 0x84f8c91e, + 0xbbd2f83d, 0xb2dcf330, 0xa9ceee27, 0xa0c0e52a, + 0x477a3cb1, 0x4e7437bc, 0x55662aab, 0x5c6821a6, + 0x63421085, 0x6a4c1b88, 0x715e069f, 0x78500d92, + 0x0f0a64d9, 0x06046fd4, 0x1d1672c3, 0x141879ce, + 0x2b3248ed, 0x223c43e0, 0x392e5ef7, 0x302055fa, + 0x9aec01b7, 0x93e20aba, 0x88f017ad, 0x81fe1ca0, + 0xbed42d83, 0xb7da268e, 0xacc83b99, 0xa5c63094, + 0xd29c59df, 0xdb9252d2, 0xc0804fc5, 0xc98e44c8, + 0xf6a475eb, 0xffaa7ee6, 0xe4b863f1, 0xedb668fc, + 0x0a0cb167, 0x0302ba6a, 0x1810a77d, 0x111eac70, + 0x2e349d53, 0x273a965e, 0x3c288b49, 0x35268044, + 0x427ce90f, 0x4b72e202, 0x5060ff15, 0x596ef418, + 0x6644c53b, 0x6f4ace36, 0x7458d321, 0x7d56d82c, + 0xa1377a0c, 0xa8397101, 0xb32b6c16, 0xba25671b, + 0x850f5638, 0x8c015d35, 0x97134022, 0x9e1d4b2f, + 0xe9472264, 0xe0492969, 0xfb5b347e, 0xf2553f73, + 0xcd7f0e50, 0xc471055d, 0xdf63184a, 0xd66d1347, + 0x31d7cadc, 0x38d9c1d1, 0x23cbdcc6, 0x2ac5d7cb, + 0x15efe6e8, 0x1ce1ede5, 0x07f3f0f2, 0x0efdfbff, + 0x79a792b4, 0x70a999b9, 0x6bbb84ae, 0x62b58fa3, + 0x5d9fbe80, 0x5491b58d, 0x4f83a89a, 0x468da397 + }, + { + 0x00000000, 0x0e0b0d09, 0x1c161a12, 0x121d171b, + 0x382c3424, 0x3627392d, 0x243a2e36, 0x2a31233f, + 0x70586848, 0x7e536541, 0x6c4e725a, 0x62457f53, + 0x48745c6c, 0x467f5165, 0x5462467e, 0x5a694b77, + 0xe0b0d090, 0xeebbdd99, 0xfca6ca82, 0xf2adc78b, + 0xd89ce4b4, 0xd697e9bd, 0xc48afea6, 0xca81f3af, + 0x90e8b8d8, 0x9ee3b5d1, 0x8cfea2ca, 0x82f5afc3, + 0xa8c48cfc, 0xa6cf81f5, 0xb4d296ee, 0xbad99be7, + 0xdb7bbb3b, 0xd570b632, 0xc76da129, 0xc966ac20, + 0xe3578f1f, 0xed5c8216, 0xff41950d, 0xf14a9804, + 0xab23d373, 0xa528de7a, 0xb735c961, 0xb93ec468, + 0x930fe757, 0x9d04ea5e, 0x8f19fd45, 0x8112f04c, + 0x3bcb6bab, 0x35c066a2, 0x27dd71b9, 0x29d67cb0, + 0x03e75f8f, 0x0dec5286, 0x1ff1459d, 0x11fa4894, + 0x4b9303e3, 0x45980eea, 0x578519f1, 0x598e14f8, + 0x73bf37c7, 0x7db43ace, 0x6fa92dd5, 0x61a220dc, + 0xadf66d76, 0xa3fd607f, 0xb1e07764, 0xbfeb7a6d, + 0x95da5952, 0x9bd1545b, 0x89cc4340, 0x87c74e49, + 0xddae053e, 0xd3a50837, 0xc1b81f2c, 0xcfb31225, + 0xe582311a, 0xeb893c13, 0xf9942b08, 0xf79f2601, + 0x4d46bde6, 0x434db0ef, 0x5150a7f4, 0x5f5baafd, + 0x756a89c2, 0x7b6184cb, 0x697c93d0, 0x67779ed9, + 0x3d1ed5ae, 0x3315d8a7, 0x2108cfbc, 0x2f03c2b5, + 0x0532e18a, 0x0b39ec83, 0x1924fb98, 0x172ff691, + 0x768dd64d, 0x7886db44, 0x6a9bcc5f, 0x6490c156, + 0x4ea1e269, 0x40aaef60, 0x52b7f87b, 0x5cbcf572, + 0x06d5be05, 0x08deb30c, 0x1ac3a417, 0x14c8a91e, + 0x3ef98a21, 0x30f28728, 0x22ef9033, 0x2ce49d3a, + 0x963d06dd, 0x98360bd4, 0x8a2b1ccf, 0x842011c6, + 0xae1132f9, 0xa01a3ff0, 0xb20728eb, 0xbc0c25e2, + 0xe6656e95, 0xe86e639c, 0xfa737487, 0xf478798e, + 0xde495ab1, 0xd04257b8, 0xc25f40a3, 0xcc544daa, + 0x41f7daec, 0x4ffcd7e5, 0x5de1c0fe, 0x53eacdf7, + 0x79dbeec8, 0x77d0e3c1, 0x65cdf4da, 0x6bc6f9d3, + 0x31afb2a4, 0x3fa4bfad, 0x2db9a8b6, 0x23b2a5bf, + 0x09838680, 0x07888b89, 0x15959c92, 0x1b9e919b, + 0xa1470a7c, 0xaf4c0775, 0xbd51106e, 0xb35a1d67, + 0x996b3e58, 0x97603351, 0x857d244a, 0x8b762943, + 0xd11f6234, 0xdf146f3d, 0xcd097826, 0xc302752f, + 0xe9335610, 0xe7385b19, 0xf5254c02, 0xfb2e410b, + 0x9a8c61d7, 0x94876cde, 0x869a7bc5, 0x889176cc, + 0xa2a055f3, 0xacab58fa, 0xbeb64fe1, 0xb0bd42e8, + 0xead4099f, 0xe4df0496, 0xf6c2138d, 0xf8c91e84, + 0xd2f83dbb, 0xdcf330b2, 0xceee27a9, 0xc0e52aa0, + 0x7a3cb147, 0x7437bc4e, 0x662aab55, 0x6821a65c, + 0x42108563, 0x4c1b886a, 0x5e069f71, 0x500d9278, + 0x0a64d90f, 0x046fd406, 0x1672c31d, 0x1879ce14, + 0x3248ed2b, 0x3c43e022, 0x2e5ef739, 0x2055fa30, + 0xec01b79a, 0xe20aba93, 0xf017ad88, 0xfe1ca081, + 0xd42d83be, 0xda268eb7, 0xc83b99ac, 0xc63094a5, + 0x9c59dfd2, 0x9252d2db, 0x804fc5c0, 0x8e44c8c9, + 0xa475ebf6, 0xaa7ee6ff, 0xb863f1e4, 0xb668fced, + 0x0cb1670a, 0x02ba6a03, 0x10a77d18, 0x1eac7011, + 0x349d532e, 0x3a965e27, 0x288b493c, 0x26804435, + 0x7ce90f42, 0x72e2024b, 0x60ff1550, 0x6ef41859, + 0x44c53b66, 0x4ace366f, 0x58d32174, 0x56d82c7d, + 0x377a0ca1, 0x397101a8, 0x2b6c16b3, 0x25671bba, + 0x0f563885, 0x015d358c, 0x13402297, 0x1d4b2f9e, + 0x472264e9, 0x492969e0, 0x5b347efb, 0x553f73f2, + 0x7f0e50cd, 0x71055dc4, 0x63184adf, 0x6d1347d6, + 0xd7cadc31, 0xd9c1d138, 0xcbdcc623, 0xc5d7cb2a, + 0xefe6e815, 0xe1ede51c, 0xf3f0f207, 0xfdfbff0e, + 0xa792b479, 0xa999b970, 0xbb84ae6b, 0xb58fa362, + 0x9fbe805d, 0x91b58d54, 0x83a89a4f, 0x8da39746 + } +}; + +#ifdef __cplusplus +} +#endif + +#endif /* _AESTAB2_H */ diff --git a/module/icp/asm-x86_64/modes/gcm_intel.S b/module/icp/asm-x86_64/modes/gcm_intel.S new file mode 100644 index 000000000000..9bb40bf239d2 --- /dev/null +++ b/module/icp/asm-x86_64/modes/gcm_intel.S @@ -0,0 +1,334 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2009 Intel Corporation + * All Rights Reserved. + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * Accelerated GHASH implementation with Intel PCLMULQDQ-NI + * instructions. This file contains an accelerated + * Galois Field Multiplication implementation. + * + * PCLMULQDQ is used to accelerate the most time-consuming part of GHASH, + * carry-less multiplication. More information about PCLMULQDQ can be + * found at: + * http://software.intel.com/en-us/articles/ + * carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/ + * + */ + +/* + * ==================================================================== + * OpenSolaris OS modifications + * + * This source originates as file galois_hash_asm.c from + * Intel Corporation dated September 21, 2009. + * + * This OpenSolaris version has these major changes from the original source: + * + * 1. Added OpenSolaris ENTRY_NP/SET_SIZE macros from + * /usr/include/sys/asm_linkage.h, lint(1B) guards, and a dummy C function + * definition for lint. + * + * 2. Formatted code, added comments, and added #includes and #defines. + * + * 3. If bit CR0.TS is set, clear and set the TS bit, after and before + * calling kpreempt_disable() and kpreempt_enable(). + * If the TS bit is not set, Save and restore %xmm registers at the beginning + * and end of function calls (%xmm* registers are not saved and restored by + * during kernel thread preemption). + * + * 4. Removed code to perform hashing. This is already done with C macro + * GHASH in gcm.c. For better performance, this removed code should be + * reintegrated in the future to replace the C GHASH macro. + * + * 5. Added code to byte swap 16-byte input and output. + * + * 6. Folded in comments from the original C source with embedded assembly + * (SB_w_shift_xor.c) + * + * 7. Renamed function and reordered parameters to match OpenSolaris: + * Intel interface: + * void galois_hash_asm(unsigned char *hk, unsigned char *s, + * unsigned char *d, int length) + * OpenSolaris OS interface: + * void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res); + * ==================================================================== + */ + + +#if defined(lint) || defined(__lint) + +#include + +/* ARGSUSED */ +void +gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res) { +} + +#else /* lint */ + +#define _ASM +#include + +#ifdef _KERNEL + /* + * Note: the CLTS macro clobbers P2 (%rsi) under i86xpv. That is, + * it calls HYPERVISOR_fpu_taskswitch() which modifies %rsi when it + * uses it to pass P2 to syscall. + * This also occurs with the STTS macro, but we dont care if + * P2 (%rsi) is modified just before function exit. + * The CLTS and STTS macros push and pop P1 (%rdi) already. + */ +#ifdef __xpv +#define PROTECTED_CLTS \ + push %rsi; \ + CLTS; \ + pop %rsi +#else +#define PROTECTED_CLTS \ + CLTS +#endif /* __xpv */ + + /* + * If CR0_TS is not set, align stack (with push %rbp) and push + * %xmm0 - %xmm10 on stack, otherwise clear CR0_TS + */ +#define CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg) \ + push %rbp; \ + mov %rsp, %rbp; \ + movq %cr0, tmpreg; \ + testq $CR0_TS, tmpreg; \ + jnz 1f; \ + and $-XMM_ALIGN, %rsp; \ + sub $[XMM_SIZE * 11], %rsp; \ + movaps %xmm0, 160(%rsp); \ + movaps %xmm1, 144(%rsp); \ + movaps %xmm2, 128(%rsp); \ + movaps %xmm3, 112(%rsp); \ + movaps %xmm4, 96(%rsp); \ + movaps %xmm5, 80(%rsp); \ + movaps %xmm6, 64(%rsp); \ + movaps %xmm7, 48(%rsp); \ + movaps %xmm8, 32(%rsp); \ + movaps %xmm9, 16(%rsp); \ + movaps %xmm10, (%rsp); \ + jmp 2f; \ +1: \ + PROTECTED_CLTS; \ +2: + + + /* + * If CR0_TS was not set above, pop %xmm0 - %xmm10 off stack, + * otherwise set CR0_TS. + */ +#define SET_TS_OR_POP_XMM_REGISTERS(tmpreg) \ + testq $CR0_TS, tmpreg; \ + jnz 1f; \ + movaps (%rsp), %xmm10; \ + movaps 16(%rsp), %xmm9; \ + movaps 32(%rsp), %xmm8; \ + movaps 48(%rsp), %xmm7; \ + movaps 64(%rsp), %xmm6; \ + movaps 80(%rsp), %xmm5; \ + movaps 96(%rsp), %xmm4; \ + movaps 112(%rsp), %xmm3; \ + movaps 128(%rsp), %xmm2; \ + movaps 144(%rsp), %xmm1; \ + movaps 160(%rsp), %xmm0; \ + jmp 2f; \ +1: \ + STTS(tmpreg); \ +2: \ + mov %rbp, %rsp; \ + pop %rbp + + +#else +#define PROTECTED_CLTS +#define CLEAR_TS_OR_PUSH_XMM_REGISTERS(tmpreg) +#define SET_TS_OR_POP_XMM_REGISTERS(tmpreg) +#endif /* _KERNEL */ + +/* + * Use this mask to byte-swap a 16-byte integer with the pshufb instruction + */ + +// static uint8_t byte_swap16_mask[] = { +// 15, 14, 13, 12, 11, 10, 9, 8, 7, 6 ,5, 4, 3, 2, 1, 0 }; +.text +.align XMM_ALIGN +.Lbyte_swap16_mask: + .byte 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 + + + +/* + * void gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res); + * + * Perform a carry-less multiplication (that is, use XOR instead of the + * multiply operator) on P1 and P2 and place the result in P3. + * + * Byte swap the input and the output. + * + * Note: x_in, y, and res all point to a block of 20-byte numbers + * (an array of two 64-bit integers). + * + * Note2: For kernel code, caller is responsible for ensuring + * kpreempt_disable() has been called. This is because %xmm registers are + * not saved/restored. Clear and set the CR0.TS bit on entry and exit, + * respectively, if TS is set on entry. Otherwise, if TS is not set, + * save and restore %xmm registers on the stack. + * + * Note3: Original Intel definition: + * void galois_hash_asm(unsigned char *hk, unsigned char *s, + * unsigned char *d, int length) + * + * Note4: Register/parameter mapping: + * Intel: + * Parameter 1: %rcx (copied to %xmm0) hk or x_in + * Parameter 2: %rdx (copied to %xmm1) s or y + * Parameter 3: %rdi (result) d or res + * OpenSolaris: + * Parameter 1: %rdi (copied to %xmm0) x_in + * Parameter 2: %rsi (copied to %xmm1) y + * Parameter 3: %rdx (result) res + */ + +ENTRY_NP(gcm_mul_pclmulqdq) + CLEAR_TS_OR_PUSH_XMM_REGISTERS(%r10) + + // + // Copy Parameters + // + movdqu (%rdi), %xmm0 // P1 + movdqu (%rsi), %xmm1 // P2 + + // + // Byte swap 16-byte input + // + lea .Lbyte_swap16_mask(%rip), %rax + movaps (%rax), %xmm10 + pshufb %xmm10, %xmm0 + pshufb %xmm10, %xmm1 + + + // + // Multiply with the hash key + // + movdqu %xmm0, %xmm3 + pclmulqdq $0, %xmm1, %xmm3 // xmm3 holds a0*b0 + + movdqu %xmm0, %xmm4 + pclmulqdq $16, %xmm1, %xmm4 // xmm4 holds a0*b1 + + movdqu %xmm0, %xmm5 + pclmulqdq $1, %xmm1, %xmm5 // xmm5 holds a1*b0 + movdqu %xmm0, %xmm6 + pclmulqdq $17, %xmm1, %xmm6 // xmm6 holds a1*b1 + + pxor %xmm5, %xmm4 // xmm4 holds a0*b1 + a1*b0 + + movdqu %xmm4, %xmm5 // move the contents of xmm4 to xmm5 + psrldq $8, %xmm4 // shift by xmm4 64 bits to the right + pslldq $8, %xmm5 // shift by xmm5 64 bits to the left + pxor %xmm5, %xmm3 + pxor %xmm4, %xmm6 // Register pair holds the result + // of the carry-less multiplication of + // xmm0 by xmm1. + + // We shift the result of the multiplication by one bit position + // to the left to cope for the fact that the bits are reversed. + movdqu %xmm3, %xmm7 + movdqu %xmm6, %xmm8 + pslld $1, %xmm3 + pslld $1, %xmm6 + psrld $31, %xmm7 + psrld $31, %xmm8 + movdqu %xmm7, %xmm9 + pslldq $4, %xmm8 + pslldq $4, %xmm7 + psrldq $12, %xmm9 + por %xmm7, %xmm3 + por %xmm8, %xmm6 + por %xmm9, %xmm6 + + // + // First phase of the reduction + // + // Move xmm3 into xmm7, xmm8, xmm9 in order to perform the shifts + // independently. + movdqu %xmm3, %xmm7 + movdqu %xmm3, %xmm8 + movdqu %xmm3, %xmm9 + pslld $31, %xmm7 // packed right shift shifting << 31 + pslld $30, %xmm8 // packed right shift shifting << 30 + pslld $25, %xmm9 // packed right shift shifting << 25 + pxor %xmm8, %xmm7 // xor the shifted versions + pxor %xmm9, %xmm7 + movdqu %xmm7, %xmm8 + pslldq $12, %xmm7 + psrldq $4, %xmm8 + pxor %xmm7, %xmm3 // first phase of the reduction complete + + // + // Second phase of the reduction + // + // Make 3 copies of xmm3 in xmm2, xmm4, xmm5 for doing these + // shift operations. + movdqu %xmm3, %xmm2 + movdqu %xmm3, %xmm4 // packed left shifting >> 1 + movdqu %xmm3, %xmm5 + psrld $1, %xmm2 + psrld $2, %xmm4 // packed left shifting >> 2 + psrld $7, %xmm5 // packed left shifting >> 7 + pxor %xmm4, %xmm2 // xor the shifted versions + pxor %xmm5, %xmm2 + pxor %xmm8, %xmm2 + pxor %xmm2, %xmm3 + pxor %xmm3, %xmm6 // the result is in xmm6 + + // + // Byte swap 16-byte result + // + pshufb %xmm10, %xmm6 // %xmm10 has the swap mask + + // + // Store the result + // + movdqu %xmm6, (%rdx) // P3 + + + // + // Cleanup and Return + // + SET_TS_OR_POP_XMM_REGISTERS(%r10) + ret + SET_SIZE(gcm_mul_pclmulqdq) + +#endif /* lint || __lint */ diff --git a/module/icp/asm-x86_64/sha1/sha1-x86_64.S b/module/icp/asm-x86_64/sha1/sha1-x86_64.S new file mode 100644 index 000000000000..53cc156a7dff --- /dev/null +++ b/module/icp/asm-x86_64/sha1/sha1-x86_64.S @@ -0,0 +1,1346 @@ +/* + * !/usr/bin/env perl + * + * ==================================================================== + * Written by Andy Polyakov for the OpenSSL + * project. The module is, however, dual licensed under OpenSSL and + * CRYPTOGAMS licenses depending on where you obtain it. For further + * details see http://www.openssl.org/~appro/cryptogams/. + * ==================================================================== + * + * sha1_block procedure for x86_64. + * + * It was brought to my attention that on EM64T compiler-generated code + * was far behind 32-bit assembler implementation. This is unlike on + * Opteron where compiler-generated code was only 15% behind 32-bit + * assembler, which originally made it hard to motivate the effort. + * There was suggestion to mechanically translate 32-bit code, but I + * dismissed it, reasoning that x86_64 offers enough register bank + * capacity to fully utilize SHA-1 parallelism. Therefore this fresh + * implementation:-) However! While 64-bit code does performs better + * on Opteron, I failed to beat 32-bit assembler on EM64T core. Well, + * x86_64 does offer larger *addressable* bank, but out-of-order core + * reaches for even more registers through dynamic aliasing, and EM64T + * core must have managed to run-time optimize even 32-bit code just as + * good as 64-bit one. Performance improvement is summarized in the + * following table: + * + * gcc 3.4 32-bit asm cycles/byte + * Opteron +45% +20% 6.8 + * Xeon P4 +65% +0% 9.9 + * Core2 +60% +10% 7.0 + * + * + * OpenSolaris OS modifications + * + * Sun elects to use this software under the BSD license. + * + * This source originates from OpenSSL file sha1-x86_64.pl at + * ftp://ftp.openssl.org/snapshot/openssl-0.9.8-stable-SNAP-20080131.tar.gz + * (presumably for future OpenSSL release 0.9.8h), with these changes: + * + * 1. Added perl "use strict" and declared variables. + * + * 2. Added OpenSolaris ENTRY_NP/SET_SIZE macros from + * /usr/include/sys/asm_linkage.h, .ident keywords, and lint(1B) guards. + * + * 3. Removed x86_64-xlate.pl script (not needed for as(1) or gas(1) + * assemblers). + * + */ + +/* + * This file was generated by a perl script (sha1-x86_64.pl). The comments from + * the original file have been pasted above. + */ + +#if defined(lint) || defined(__lint) +#include +#include + +/* ARGSUSED */ +void +sha1_block_data_order(SHA1_CTX *ctx, const void *inpp, size_t blocks) +{ +} + +#else +#define _ASM +#include +ENTRY_NP(sha1_block_data_order) + push %rbx + push %rbp + push %r12 + mov %rsp,%rax + mov %rdi,%r8 # reassigned argument + sub $72,%rsp + mov %rsi,%r9 # reassigned argument + and $-64,%rsp + mov %rdx,%r10 # reassigned argument + mov %rax,64(%rsp) + + mov 0(%r8),%edx + mov 4(%r8),%esi + mov 8(%r8),%edi + mov 12(%r8),%ebp + mov 16(%r8),%r11d +.align 4 +.Lloop: + mov 0(%r9),%eax + bswap %eax + mov %eax,0(%rsp) + lea 0x5a827999(%eax,%r11d),%r12d + mov %edi,%ebx + mov 4(%r9),%eax + mov %edx,%r11d + xor %ebp,%ebx + bswap %eax + rol $5,%r11d + and %esi,%ebx + mov %eax,4(%rsp) + add %r11d,%r12d + xor %ebp,%ebx + rol $30,%esi + add %ebx,%r12d + lea 0x5a827999(%eax,%ebp),%r11d + mov %esi,%ebx + mov 8(%r9),%eax + mov %r12d,%ebp + xor %edi,%ebx + bswap %eax + rol $5,%ebp + and %edx,%ebx + mov %eax,8(%rsp) + add %ebp,%r11d + xor %edi,%ebx + rol $30,%edx + add %ebx,%r11d + lea 0x5a827999(%eax,%edi),%ebp + mov %edx,%ebx + mov 12(%r9),%eax + mov %r11d,%edi + xor %esi,%ebx + bswap %eax + rol $5,%edi + and %r12d,%ebx + mov %eax,12(%rsp) + add %edi,%ebp + xor %esi,%ebx + rol $30,%r12d + add %ebx,%ebp + lea 0x5a827999(%eax,%esi),%edi + mov %r12d,%ebx + mov 16(%r9),%eax + mov %ebp,%esi + xor %edx,%ebx + bswap %eax + rol $5,%esi + and %r11d,%ebx + mov %eax,16(%rsp) + add %esi,%edi + xor %edx,%ebx + rol $30,%r11d + add %ebx,%edi + lea 0x5a827999(%eax,%edx),%esi + mov %r11d,%ebx + mov 20(%r9),%eax + mov %edi,%edx + xor %r12d,%ebx + bswap %eax + rol $5,%edx + and %ebp,%ebx + mov %eax,20(%rsp) + add %edx,%esi + xor %r12d,%ebx + rol $30,%ebp + add %ebx,%esi + lea 0x5a827999(%eax,%r12d),%edx + mov %ebp,%ebx + mov 24(%r9),%eax + mov %esi,%r12d + xor %r11d,%ebx + bswap %eax + rol $5,%r12d + and %edi,%ebx + mov %eax,24(%rsp) + add %r12d,%edx + xor %r11d,%ebx + rol $30,%edi + add %ebx,%edx + lea 0x5a827999(%eax,%r11d),%r12d + mov %edi,%ebx + mov 28(%r9),%eax + mov %edx,%r11d + xor %ebp,%ebx + bswap %eax + rol $5,%r11d + and %esi,%ebx + mov %eax,28(%rsp) + add %r11d,%r12d + xor %ebp,%ebx + rol $30,%esi + add %ebx,%r12d + lea 0x5a827999(%eax,%ebp),%r11d + mov %esi,%ebx + mov 32(%r9),%eax + mov %r12d,%ebp + xor %edi,%ebx + bswap %eax + rol $5,%ebp + and %edx,%ebx + mov %eax,32(%rsp) + add %ebp,%r11d + xor %edi,%ebx + rol $30,%edx + add %ebx,%r11d + lea 0x5a827999(%eax,%edi),%ebp + mov %edx,%ebx + mov 36(%r9),%eax + mov %r11d,%edi + xor %esi,%ebx + bswap %eax + rol $5,%edi + and %r12d,%ebx + mov %eax,36(%rsp) + add %edi,%ebp + xor %esi,%ebx + rol $30,%r12d + add %ebx,%ebp + lea 0x5a827999(%eax,%esi),%edi + mov %r12d,%ebx + mov 40(%r9),%eax + mov %ebp,%esi + xor %edx,%ebx + bswap %eax + rol $5,%esi + and %r11d,%ebx + mov %eax,40(%rsp) + add %esi,%edi + xor %edx,%ebx + rol $30,%r11d + add %ebx,%edi + lea 0x5a827999(%eax,%edx),%esi + mov %r11d,%ebx + mov 44(%r9),%eax + mov %edi,%edx + xor %r12d,%ebx + bswap %eax + rol $5,%edx + and %ebp,%ebx + mov %eax,44(%rsp) + add %edx,%esi + xor %r12d,%ebx + rol $30,%ebp + add %ebx,%esi + lea 0x5a827999(%eax,%r12d),%edx + mov %ebp,%ebx + mov 48(%r9),%eax + mov %esi,%r12d + xor %r11d,%ebx + bswap %eax + rol $5,%r12d + and %edi,%ebx + mov %eax,48(%rsp) + add %r12d,%edx + xor %r11d,%ebx + rol $30,%edi + add %ebx,%edx + lea 0x5a827999(%eax,%r11d),%r12d + mov %edi,%ebx + mov 52(%r9),%eax + mov %edx,%r11d + xor %ebp,%ebx + bswap %eax + rol $5,%r11d + and %esi,%ebx + mov %eax,52(%rsp) + add %r11d,%r12d + xor %ebp,%ebx + rol $30,%esi + add %ebx,%r12d + lea 0x5a827999(%eax,%ebp),%r11d + mov %esi,%ebx + mov 56(%r9),%eax + mov %r12d,%ebp + xor %edi,%ebx + bswap %eax + rol $5,%ebp + and %edx,%ebx + mov %eax,56(%rsp) + add %ebp,%r11d + xor %edi,%ebx + rol $30,%edx + add %ebx,%r11d + lea 0x5a827999(%eax,%edi),%ebp + mov %edx,%ebx + mov 60(%r9),%eax + mov %r11d,%edi + xor %esi,%ebx + bswap %eax + rol $5,%edi + and %r12d,%ebx + mov %eax,60(%rsp) + add %edi,%ebp + xor %esi,%ebx + rol $30,%r12d + add %ebx,%ebp + lea 0x5a827999(%eax,%esi),%edi + mov 0(%rsp),%eax + mov %r12d,%ebx + mov %ebp,%esi + xor 8(%rsp),%eax + xor %edx,%ebx + rol $5,%esi + xor 32(%rsp),%eax + and %r11d,%ebx + add %esi,%edi + xor 52(%rsp),%eax + xor %edx,%ebx + rol $30,%r11d + add %ebx,%edi + rol $1,%eax + mov %eax,0(%rsp) + lea 0x5a827999(%eax,%edx),%esi + mov 4(%rsp),%eax + mov %r11d,%ebx + mov %edi,%edx + xor 12(%rsp),%eax + xor %r12d,%ebx + rol $5,%edx + xor 36(%rsp),%eax + and %ebp,%ebx + add %edx,%esi + xor 56(%rsp),%eax + xor %r12d,%ebx + rol $30,%ebp + add %ebx,%esi + rol $1,%eax + mov %eax,4(%rsp) + lea 0x5a827999(%eax,%r12d),%edx + mov 8(%rsp),%eax + mov %ebp,%ebx + mov %esi,%r12d + xor 16(%rsp),%eax + xor %r11d,%ebx + rol $5,%r12d + xor 40(%rsp),%eax + and %edi,%ebx + add %r12d,%edx + xor 60(%rsp),%eax + xor %r11d,%ebx + rol $30,%edi + add %ebx,%edx + rol $1,%eax + mov %eax,8(%rsp) + lea 0x5a827999(%eax,%r11d),%r12d + mov 12(%rsp),%eax + mov %edi,%ebx + mov %edx,%r11d + xor 20(%rsp),%eax + xor %ebp,%ebx + rol $5,%r11d + xor 44(%rsp),%eax + and %esi,%ebx + add %r11d,%r12d + xor 0(%rsp),%eax + xor %ebp,%ebx + rol $30,%esi + add %ebx,%r12d + rol $1,%eax + mov %eax,12(%rsp) + lea 0x5a827999(%eax,%ebp),%r11d + mov 16(%rsp),%eax + mov %esi,%ebx + mov %r12d,%ebp + xor 24(%rsp),%eax + xor %edi,%ebx + rol $5,%ebp + xor 48(%rsp),%eax + and %edx,%ebx + add %ebp,%r11d + xor 4(%rsp),%eax + xor %edi,%ebx + rol $30,%edx + add %ebx,%r11d + rol $1,%eax + mov %eax,16(%rsp) + lea 0x6ed9eba1(%eax,%edi),%ebp + mov 20(%rsp),%eax + mov %edx,%ebx + mov %r11d,%edi + xor 28(%rsp),%eax + xor %r12d,%ebx + rol $5,%edi + xor 52(%rsp),%eax + xor %esi,%ebx + add %edi,%ebp + xor 8(%rsp),%eax + rol $30,%r12d + add %ebx,%ebp + rol $1,%eax + mov %eax,20(%rsp) + lea 0x6ed9eba1(%eax,%esi),%edi + mov 24(%rsp),%eax + mov %r12d,%ebx + mov %ebp,%esi + xor 32(%rsp),%eax + xor %r11d,%ebx + rol $5,%esi + xor 56(%rsp),%eax + xor %edx,%ebx + add %esi,%edi + xor 12(%rsp),%eax + rol $30,%r11d + add %ebx,%edi + rol $1,%eax + mov %eax,24(%rsp) + lea 0x6ed9eba1(%eax,%edx),%esi + mov 28(%rsp),%eax + mov %r11d,%ebx + mov %edi,%edx + xor 36(%rsp),%eax + xor %ebp,%ebx + rol $5,%edx + xor 60(%rsp),%eax + xor %r12d,%ebx + add %edx,%esi + xor 16(%rsp),%eax + rol $30,%ebp + add %ebx,%esi + rol $1,%eax + mov %eax,28(%rsp) + lea 0x6ed9eba1(%eax,%r12d),%edx + mov 32(%rsp),%eax + mov %ebp,%ebx + mov %esi,%r12d + xor 40(%rsp),%eax + xor %edi,%ebx + rol $5,%r12d + xor 0(%rsp),%eax + xor %r11d,%ebx + add %r12d,%edx + xor 20(%rsp),%eax + rol $30,%edi + add %ebx,%edx + rol $1,%eax + mov %eax,32(%rsp) + lea 0x6ed9eba1(%eax,%r11d),%r12d + mov 36(%rsp),%eax + mov %edi,%ebx + mov %edx,%r11d + xor 44(%rsp),%eax + xor %esi,%ebx + rol $5,%r11d + xor 4(%rsp),%eax + xor %ebp,%ebx + add %r11d,%r12d + xor 24(%rsp),%eax + rol $30,%esi + add %ebx,%r12d + rol $1,%eax + mov %eax,36(%rsp) + lea 0x6ed9eba1(%eax,%ebp),%r11d + mov 40(%rsp),%eax + mov %esi,%ebx + mov %r12d,%ebp + xor 48(%rsp),%eax + xor %edx,%ebx + rol $5,%ebp + xor 8(%rsp),%eax + xor %edi,%ebx + add %ebp,%r11d + xor 28(%rsp),%eax + rol $30,%edx + add %ebx,%r11d + rol $1,%eax + mov %eax,40(%rsp) + lea 0x6ed9eba1(%eax,%edi),%ebp + mov 44(%rsp),%eax + mov %edx,%ebx + mov %r11d,%edi + xor 52(%rsp),%eax + xor %r12d,%ebx + rol $5,%edi + xor 12(%rsp),%eax + xor %esi,%ebx + add %edi,%ebp + xor 32(%rsp),%eax + rol $30,%r12d + add %ebx,%ebp + rol $1,%eax + mov %eax,44(%rsp) + lea 0x6ed9eba1(%eax,%esi),%edi + mov 48(%rsp),%eax + mov %r12d,%ebx + mov %ebp,%esi + xor 56(%rsp),%eax + xor %r11d,%ebx + rol $5,%esi + xor 16(%rsp),%eax + xor %edx,%ebx + add %esi,%edi + xor 36(%rsp),%eax + rol $30,%r11d + add %ebx,%edi + rol $1,%eax + mov %eax,48(%rsp) + lea 0x6ed9eba1(%eax,%edx),%esi + mov 52(%rsp),%eax + mov %r11d,%ebx + mov %edi,%edx + xor 60(%rsp),%eax + xor %ebp,%ebx + rol $5,%edx + xor 20(%rsp),%eax + xor %r12d,%ebx + add %edx,%esi + xor 40(%rsp),%eax + rol $30,%ebp + add %ebx,%esi + rol $1,%eax + mov %eax,52(%rsp) + lea 0x6ed9eba1(%eax,%r12d),%edx + mov 56(%rsp),%eax + mov %ebp,%ebx + mov %esi,%r12d + xor 0(%rsp),%eax + xor %edi,%ebx + rol $5,%r12d + xor 24(%rsp),%eax + xor %r11d,%ebx + add %r12d,%edx + xor 44(%rsp),%eax + rol $30,%edi + add %ebx,%edx + rol $1,%eax + mov %eax,56(%rsp) + lea 0x6ed9eba1(%eax,%r11d),%r12d + mov 60(%rsp),%eax + mov %edi,%ebx + mov %edx,%r11d + xor 4(%rsp),%eax + xor %esi,%ebx + rol $5,%r11d + xor 28(%rsp),%eax + xor %ebp,%ebx + add %r11d,%r12d + xor 48(%rsp),%eax + rol $30,%esi + add %ebx,%r12d + rol $1,%eax + mov %eax,60(%rsp) + lea 0x6ed9eba1(%eax,%ebp),%r11d + mov 0(%rsp),%eax + mov %esi,%ebx + mov %r12d,%ebp + xor 8(%rsp),%eax + xor %edx,%ebx + rol $5,%ebp + xor 32(%rsp),%eax + xor %edi,%ebx + add %ebp,%r11d + xor 52(%rsp),%eax + rol $30,%edx + add %ebx,%r11d + rol $1,%eax + mov %eax,0(%rsp) + lea 0x6ed9eba1(%eax,%edi),%ebp + mov 4(%rsp),%eax + mov %edx,%ebx + mov %r11d,%edi + xor 12(%rsp),%eax + xor %r12d,%ebx + rol $5,%edi + xor 36(%rsp),%eax + xor %esi,%ebx + add %edi,%ebp + xor 56(%rsp),%eax + rol $30,%r12d + add %ebx,%ebp + rol $1,%eax + mov %eax,4(%rsp) + lea 0x6ed9eba1(%eax,%esi),%edi + mov 8(%rsp),%eax + mov %r12d,%ebx + mov %ebp,%esi + xor 16(%rsp),%eax + xor %r11d,%ebx + rol $5,%esi + xor 40(%rsp),%eax + xor %edx,%ebx + add %esi,%edi + xor 60(%rsp),%eax + rol $30,%r11d + add %ebx,%edi + rol $1,%eax + mov %eax,8(%rsp) + lea 0x6ed9eba1(%eax,%edx),%esi + mov 12(%rsp),%eax + mov %r11d,%ebx + mov %edi,%edx + xor 20(%rsp),%eax + xor %ebp,%ebx + rol $5,%edx + xor 44(%rsp),%eax + xor %r12d,%ebx + add %edx,%esi + xor 0(%rsp),%eax + rol $30,%ebp + add %ebx,%esi + rol $1,%eax + mov %eax,12(%rsp) + lea 0x6ed9eba1(%eax,%r12d),%edx + mov 16(%rsp),%eax + mov %ebp,%ebx + mov %esi,%r12d + xor 24(%rsp),%eax + xor %edi,%ebx + rol $5,%r12d + xor 48(%rsp),%eax + xor %r11d,%ebx + add %r12d,%edx + xor 4(%rsp),%eax + rol $30,%edi + add %ebx,%edx + rol $1,%eax + mov %eax,16(%rsp) + lea 0x6ed9eba1(%eax,%r11d),%r12d + mov 20(%rsp),%eax + mov %edi,%ebx + mov %edx,%r11d + xor 28(%rsp),%eax + xor %esi,%ebx + rol $5,%r11d + xor 52(%rsp),%eax + xor %ebp,%ebx + add %r11d,%r12d + xor 8(%rsp),%eax + rol $30,%esi + add %ebx,%r12d + rol $1,%eax + mov %eax,20(%rsp) + lea 0x6ed9eba1(%eax,%ebp),%r11d + mov 24(%rsp),%eax + mov %esi,%ebx + mov %r12d,%ebp + xor 32(%rsp),%eax + xor %edx,%ebx + rol $5,%ebp + xor 56(%rsp),%eax + xor %edi,%ebx + add %ebp,%r11d + xor 12(%rsp),%eax + rol $30,%edx + add %ebx,%r11d + rol $1,%eax + mov %eax,24(%rsp) + lea 0x6ed9eba1(%eax,%edi),%ebp + mov 28(%rsp),%eax + mov %edx,%ebx + mov %r11d,%edi + xor 36(%rsp),%eax + xor %r12d,%ebx + rol $5,%edi + xor 60(%rsp),%eax + xor %esi,%ebx + add %edi,%ebp + xor 16(%rsp),%eax + rol $30,%r12d + add %ebx,%ebp + rol $1,%eax + mov %eax,28(%rsp) + lea 0x6ed9eba1(%eax,%esi),%edi + mov 32(%rsp),%eax + mov %r12d,%ebx + mov %ebp,%esi + xor 40(%rsp),%eax + xor %r11d,%ebx + rol $5,%esi + xor 0(%rsp),%eax + xor %edx,%ebx + add %esi,%edi + xor 20(%rsp),%eax + rol $30,%r11d + add %ebx,%edi + rol $1,%eax + mov %eax,32(%rsp) + lea -0x70e44324(%eax,%edx),%esi + mov 36(%rsp),%eax + mov %ebp,%ebx + mov %ebp,%ecx + xor 44(%rsp),%eax + mov %edi,%edx + and %r11d,%ebx + xor 4(%rsp),%eax + or %r11d,%ecx + rol $5,%edx + xor 24(%rsp),%eax + and %r12d,%ecx + add %edx,%esi + rol $1,%eax + or %ecx,%ebx + rol $30,%ebp + mov %eax,36(%rsp) + add %ebx,%esi + lea -0x70e44324(%eax,%r12d),%edx + mov 40(%rsp),%eax + mov %edi,%ebx + mov %edi,%ecx + xor 48(%rsp),%eax + mov %esi,%r12d + and %ebp,%ebx + xor 8(%rsp),%eax + or %ebp,%ecx + rol $5,%r12d + xor 28(%rsp),%eax + and %r11d,%ecx + add %r12d,%edx + rol $1,%eax + or %ecx,%ebx + rol $30,%edi + mov %eax,40(%rsp) + add %ebx,%edx + lea -0x70e44324(%eax,%r11d),%r12d + mov 44(%rsp),%eax + mov %esi,%ebx + mov %esi,%ecx + xor 52(%rsp),%eax + mov %edx,%r11d + and %edi,%ebx + xor 12(%rsp),%eax + or %edi,%ecx + rol $5,%r11d + xor 32(%rsp),%eax + and %ebp,%ecx + add %r11d,%r12d + rol $1,%eax + or %ecx,%ebx + rol $30,%esi + mov %eax,44(%rsp) + add %ebx,%r12d + lea -0x70e44324(%eax,%ebp),%r11d + mov 48(%rsp),%eax + mov %edx,%ebx + mov %edx,%ecx + xor 56(%rsp),%eax + mov %r12d,%ebp + and %esi,%ebx + xor 16(%rsp),%eax + or %esi,%ecx + rol $5,%ebp + xor 36(%rsp),%eax + and %edi,%ecx + add %ebp,%r11d + rol $1,%eax + or %ecx,%ebx + rol $30,%edx + mov %eax,48(%rsp) + add %ebx,%r11d + lea -0x70e44324(%eax,%edi),%ebp + mov 52(%rsp),%eax + mov %r12d,%ebx + mov %r12d,%ecx + xor 60(%rsp),%eax + mov %r11d,%edi + and %edx,%ebx + xor 20(%rsp),%eax + or %edx,%ecx + rol $5,%edi + xor 40(%rsp),%eax + and %esi,%ecx + add %edi,%ebp + rol $1,%eax + or %ecx,%ebx + rol $30,%r12d + mov %eax,52(%rsp) + add %ebx,%ebp + lea -0x70e44324(%eax,%esi),%edi + mov 56(%rsp),%eax + mov %r11d,%ebx + mov %r11d,%ecx + xor 0(%rsp),%eax + mov %ebp,%esi + and %r12d,%ebx + xor 24(%rsp),%eax + or %r12d,%ecx + rol $5,%esi + xor 44(%rsp),%eax + and %edx,%ecx + add %esi,%edi + rol $1,%eax + or %ecx,%ebx + rol $30,%r11d + mov %eax,56(%rsp) + add %ebx,%edi + lea -0x70e44324(%eax,%edx),%esi + mov 60(%rsp),%eax + mov %ebp,%ebx + mov %ebp,%ecx + xor 4(%rsp),%eax + mov %edi,%edx + and %r11d,%ebx + xor 28(%rsp),%eax + or %r11d,%ecx + rol $5,%edx + xor 48(%rsp),%eax + and %r12d,%ecx + add %edx,%esi + rol $1,%eax + or %ecx,%ebx + rol $30,%ebp + mov %eax,60(%rsp) + add %ebx,%esi + lea -0x70e44324(%eax,%r12d),%edx + mov 0(%rsp),%eax + mov %edi,%ebx + mov %edi,%ecx + xor 8(%rsp),%eax + mov %esi,%r12d + and %ebp,%ebx + xor 32(%rsp),%eax + or %ebp,%ecx + rol $5,%r12d + xor 52(%rsp),%eax + and %r11d,%ecx + add %r12d,%edx + rol $1,%eax + or %ecx,%ebx + rol $30,%edi + mov %eax,0(%rsp) + add %ebx,%edx + lea -0x70e44324(%eax,%r11d),%r12d + mov 4(%rsp),%eax + mov %esi,%ebx + mov %esi,%ecx + xor 12(%rsp),%eax + mov %edx,%r11d + and %edi,%ebx + xor 36(%rsp),%eax + or %edi,%ecx + rol $5,%r11d + xor 56(%rsp),%eax + and %ebp,%ecx + add %r11d,%r12d + rol $1,%eax + or %ecx,%ebx + rol $30,%esi + mov %eax,4(%rsp) + add %ebx,%r12d + lea -0x70e44324(%eax,%ebp),%r11d + mov 8(%rsp),%eax + mov %edx,%ebx + mov %edx,%ecx + xor 16(%rsp),%eax + mov %r12d,%ebp + and %esi,%ebx + xor 40(%rsp),%eax + or %esi,%ecx + rol $5,%ebp + xor 60(%rsp),%eax + and %edi,%ecx + add %ebp,%r11d + rol $1,%eax + or %ecx,%ebx + rol $30,%edx + mov %eax,8(%rsp) + add %ebx,%r11d + lea -0x70e44324(%eax,%edi),%ebp + mov 12(%rsp),%eax + mov %r12d,%ebx + mov %r12d,%ecx + xor 20(%rsp),%eax + mov %r11d,%edi + and %edx,%ebx + xor 44(%rsp),%eax + or %edx,%ecx + rol $5,%edi + xor 0(%rsp),%eax + and %esi,%ecx + add %edi,%ebp + rol $1,%eax + or %ecx,%ebx + rol $30,%r12d + mov %eax,12(%rsp) + add %ebx,%ebp + lea -0x70e44324(%eax,%esi),%edi + mov 16(%rsp),%eax + mov %r11d,%ebx + mov %r11d,%ecx + xor 24(%rsp),%eax + mov %ebp,%esi + and %r12d,%ebx + xor 48(%rsp),%eax + or %r12d,%ecx + rol $5,%esi + xor 4(%rsp),%eax + and %edx,%ecx + add %esi,%edi + rol $1,%eax + or %ecx,%ebx + rol $30,%r11d + mov %eax,16(%rsp) + add %ebx,%edi + lea -0x70e44324(%eax,%edx),%esi + mov 20(%rsp),%eax + mov %ebp,%ebx + mov %ebp,%ecx + xor 28(%rsp),%eax + mov %edi,%edx + and %r11d,%ebx + xor 52(%rsp),%eax + or %r11d,%ecx + rol $5,%edx + xor 8(%rsp),%eax + and %r12d,%ecx + add %edx,%esi + rol $1,%eax + or %ecx,%ebx + rol $30,%ebp + mov %eax,20(%rsp) + add %ebx,%esi + lea -0x70e44324(%eax,%r12d),%edx + mov 24(%rsp),%eax + mov %edi,%ebx + mov %edi,%ecx + xor 32(%rsp),%eax + mov %esi,%r12d + and %ebp,%ebx + xor 56(%rsp),%eax + or %ebp,%ecx + rol $5,%r12d + xor 12(%rsp),%eax + and %r11d,%ecx + add %r12d,%edx + rol $1,%eax + or %ecx,%ebx + rol $30,%edi + mov %eax,24(%rsp) + add %ebx,%edx + lea -0x70e44324(%eax,%r11d),%r12d + mov 28(%rsp),%eax + mov %esi,%ebx + mov %esi,%ecx + xor 36(%rsp),%eax + mov %edx,%r11d + and %edi,%ebx + xor 60(%rsp),%eax + or %edi,%ecx + rol $5,%r11d + xor 16(%rsp),%eax + and %ebp,%ecx + add %r11d,%r12d + rol $1,%eax + or %ecx,%ebx + rol $30,%esi + mov %eax,28(%rsp) + add %ebx,%r12d + lea -0x70e44324(%eax,%ebp),%r11d + mov 32(%rsp),%eax + mov %edx,%ebx + mov %edx,%ecx + xor 40(%rsp),%eax + mov %r12d,%ebp + and %esi,%ebx + xor 0(%rsp),%eax + or %esi,%ecx + rol $5,%ebp + xor 20(%rsp),%eax + and %edi,%ecx + add %ebp,%r11d + rol $1,%eax + or %ecx,%ebx + rol $30,%edx + mov %eax,32(%rsp) + add %ebx,%r11d + lea -0x70e44324(%eax,%edi),%ebp + mov 36(%rsp),%eax + mov %r12d,%ebx + mov %r12d,%ecx + xor 44(%rsp),%eax + mov %r11d,%edi + and %edx,%ebx + xor 4(%rsp),%eax + or %edx,%ecx + rol $5,%edi + xor 24(%rsp),%eax + and %esi,%ecx + add %edi,%ebp + rol $1,%eax + or %ecx,%ebx + rol $30,%r12d + mov %eax,36(%rsp) + add %ebx,%ebp + lea -0x70e44324(%eax,%esi),%edi + mov 40(%rsp),%eax + mov %r11d,%ebx + mov %r11d,%ecx + xor 48(%rsp),%eax + mov %ebp,%esi + and %r12d,%ebx + xor 8(%rsp),%eax + or %r12d,%ecx + rol $5,%esi + xor 28(%rsp),%eax + and %edx,%ecx + add %esi,%edi + rol $1,%eax + or %ecx,%ebx + rol $30,%r11d + mov %eax,40(%rsp) + add %ebx,%edi + lea -0x70e44324(%eax,%edx),%esi + mov 44(%rsp),%eax + mov %ebp,%ebx + mov %ebp,%ecx + xor 52(%rsp),%eax + mov %edi,%edx + and %r11d,%ebx + xor 12(%rsp),%eax + or %r11d,%ecx + rol $5,%edx + xor 32(%rsp),%eax + and %r12d,%ecx + add %edx,%esi + rol $1,%eax + or %ecx,%ebx + rol $30,%ebp + mov %eax,44(%rsp) + add %ebx,%esi + lea -0x70e44324(%eax,%r12d),%edx + mov 48(%rsp),%eax + mov %edi,%ebx + mov %edi,%ecx + xor 56(%rsp),%eax + mov %esi,%r12d + and %ebp,%ebx + xor 16(%rsp),%eax + or %ebp,%ecx + rol $5,%r12d + xor 36(%rsp),%eax + and %r11d,%ecx + add %r12d,%edx + rol $1,%eax + or %ecx,%ebx + rol $30,%edi + mov %eax,48(%rsp) + add %ebx,%edx + lea -0x359d3e2a(%eax,%r11d),%r12d + mov 52(%rsp),%eax + mov %edi,%ebx + mov %edx,%r11d + xor 60(%rsp),%eax + xor %esi,%ebx + rol $5,%r11d + xor 20(%rsp),%eax + xor %ebp,%ebx + add %r11d,%r12d + xor 40(%rsp),%eax + rol $30,%esi + add %ebx,%r12d + rol $1,%eax + mov %eax,52(%rsp) + lea -0x359d3e2a(%eax,%ebp),%r11d + mov 56(%rsp),%eax + mov %esi,%ebx + mov %r12d,%ebp + xor 0(%rsp),%eax + xor %edx,%ebx + rol $5,%ebp + xor 24(%rsp),%eax + xor %edi,%ebx + add %ebp,%r11d + xor 44(%rsp),%eax + rol $30,%edx + add %ebx,%r11d + rol $1,%eax + mov %eax,56(%rsp) + lea -0x359d3e2a(%eax,%edi),%ebp + mov 60(%rsp),%eax + mov %edx,%ebx + mov %r11d,%edi + xor 4(%rsp),%eax + xor %r12d,%ebx + rol $5,%edi + xor 28(%rsp),%eax + xor %esi,%ebx + add %edi,%ebp + xor 48(%rsp),%eax + rol $30,%r12d + add %ebx,%ebp + rol $1,%eax + mov %eax,60(%rsp) + lea -0x359d3e2a(%eax,%esi),%edi + mov 0(%rsp),%eax + mov %r12d,%ebx + mov %ebp,%esi + xor 8(%rsp),%eax + xor %r11d,%ebx + rol $5,%esi + xor 32(%rsp),%eax + xor %edx,%ebx + add %esi,%edi + xor 52(%rsp),%eax + rol $30,%r11d + add %ebx,%edi + rol $1,%eax + mov %eax,0(%rsp) + lea -0x359d3e2a(%eax,%edx),%esi + mov 4(%rsp),%eax + mov %r11d,%ebx + mov %edi,%edx + xor 12(%rsp),%eax + xor %ebp,%ebx + rol $5,%edx + xor 36(%rsp),%eax + xor %r12d,%ebx + add %edx,%esi + xor 56(%rsp),%eax + rol $30,%ebp + add %ebx,%esi + rol $1,%eax + mov %eax,4(%rsp) + lea -0x359d3e2a(%eax,%r12d),%edx + mov 8(%rsp),%eax + mov %ebp,%ebx + mov %esi,%r12d + xor 16(%rsp),%eax + xor %edi,%ebx + rol $5,%r12d + xor 40(%rsp),%eax + xor %r11d,%ebx + add %r12d,%edx + xor 60(%rsp),%eax + rol $30,%edi + add %ebx,%edx + rol $1,%eax + mov %eax,8(%rsp) + lea -0x359d3e2a(%eax,%r11d),%r12d + mov 12(%rsp),%eax + mov %edi,%ebx + mov %edx,%r11d + xor 20(%rsp),%eax + xor %esi,%ebx + rol $5,%r11d + xor 44(%rsp),%eax + xor %ebp,%ebx + add %r11d,%r12d + xor 0(%rsp),%eax + rol $30,%esi + add %ebx,%r12d + rol $1,%eax + mov %eax,12(%rsp) + lea -0x359d3e2a(%eax,%ebp),%r11d + mov 16(%rsp),%eax + mov %esi,%ebx + mov %r12d,%ebp + xor 24(%rsp),%eax + xor %edx,%ebx + rol $5,%ebp + xor 48(%rsp),%eax + xor %edi,%ebx + add %ebp,%r11d + xor 4(%rsp),%eax + rol $30,%edx + add %ebx,%r11d + rol $1,%eax + mov %eax,16(%rsp) + lea -0x359d3e2a(%eax,%edi),%ebp + mov 20(%rsp),%eax + mov %edx,%ebx + mov %r11d,%edi + xor 28(%rsp),%eax + xor %r12d,%ebx + rol $5,%edi + xor 52(%rsp),%eax + xor %esi,%ebx + add %edi,%ebp + xor 8(%rsp),%eax + rol $30,%r12d + add %ebx,%ebp + rol $1,%eax + mov %eax,20(%rsp) + lea -0x359d3e2a(%eax,%esi),%edi + mov 24(%rsp),%eax + mov %r12d,%ebx + mov %ebp,%esi + xor 32(%rsp),%eax + xor %r11d,%ebx + rol $5,%esi + xor 56(%rsp),%eax + xor %edx,%ebx + add %esi,%edi + xor 12(%rsp),%eax + rol $30,%r11d + add %ebx,%edi + rol $1,%eax + mov %eax,24(%rsp) + lea -0x359d3e2a(%eax,%edx),%esi + mov 28(%rsp),%eax + mov %r11d,%ebx + mov %edi,%edx + xor 36(%rsp),%eax + xor %ebp,%ebx + rol $5,%edx + xor 60(%rsp),%eax + xor %r12d,%ebx + add %edx,%esi + xor 16(%rsp),%eax + rol $30,%ebp + add %ebx,%esi + rol $1,%eax + mov %eax,28(%rsp) + lea -0x359d3e2a(%eax,%r12d),%edx + mov 32(%rsp),%eax + mov %ebp,%ebx + mov %esi,%r12d + xor 40(%rsp),%eax + xor %edi,%ebx + rol $5,%r12d + xor 0(%rsp),%eax + xor %r11d,%ebx + add %r12d,%edx + xor 20(%rsp),%eax + rol $30,%edi + add %ebx,%edx + rol $1,%eax + mov %eax,32(%rsp) + lea -0x359d3e2a(%eax,%r11d),%r12d + mov 36(%rsp),%eax + mov %edi,%ebx + mov %edx,%r11d + xor 44(%rsp),%eax + xor %esi,%ebx + rol $5,%r11d + xor 4(%rsp),%eax + xor %ebp,%ebx + add %r11d,%r12d + xor 24(%rsp),%eax + rol $30,%esi + add %ebx,%r12d + rol $1,%eax + mov %eax,36(%rsp) + lea -0x359d3e2a(%eax,%ebp),%r11d + mov 40(%rsp),%eax + mov %esi,%ebx + mov %r12d,%ebp + xor 48(%rsp),%eax + xor %edx,%ebx + rol $5,%ebp + xor 8(%rsp),%eax + xor %edi,%ebx + add %ebp,%r11d + xor 28(%rsp),%eax + rol $30,%edx + add %ebx,%r11d + rol $1,%eax + mov %eax,40(%rsp) + lea -0x359d3e2a(%eax,%edi),%ebp + mov 44(%rsp),%eax + mov %edx,%ebx + mov %r11d,%edi + xor 52(%rsp),%eax + xor %r12d,%ebx + rol $5,%edi + xor 12(%rsp),%eax + xor %esi,%ebx + add %edi,%ebp + xor 32(%rsp),%eax + rol $30,%r12d + add %ebx,%ebp + rol $1,%eax + mov %eax,44(%rsp) + lea -0x359d3e2a(%eax,%esi),%edi + mov 48(%rsp),%eax + mov %r12d,%ebx + mov %ebp,%esi + xor 56(%rsp),%eax + xor %r11d,%ebx + rol $5,%esi + xor 16(%rsp),%eax + xor %edx,%ebx + add %esi,%edi + xor 36(%rsp),%eax + rol $30,%r11d + add %ebx,%edi + rol $1,%eax + mov %eax,48(%rsp) + lea -0x359d3e2a(%eax,%edx),%esi + mov 52(%rsp),%eax + mov %r11d,%ebx + mov %edi,%edx + xor 60(%rsp),%eax + xor %ebp,%ebx + rol $5,%edx + xor 20(%rsp),%eax + xor %r12d,%ebx + add %edx,%esi + xor 40(%rsp),%eax + rol $30,%ebp + add %ebx,%esi + rol $1,%eax + lea -0x359d3e2a(%eax,%r12d),%edx + mov 56(%rsp),%eax + mov %ebp,%ebx + mov %esi,%r12d + xor 0(%rsp),%eax + xor %edi,%ebx + rol $5,%r12d + xor 24(%rsp),%eax + xor %r11d,%ebx + add %r12d,%edx + xor 44(%rsp),%eax + rol $30,%edi + add %ebx,%edx + rol $1,%eax + lea -0x359d3e2a(%eax,%r11d),%r12d + mov 60(%rsp),%eax + mov %edi,%ebx + mov %edx,%r11d + xor 4(%rsp),%eax + xor %esi,%ebx + rol $5,%r11d + xor 28(%rsp),%eax + xor %ebp,%ebx + add %r11d,%r12d + xor 48(%rsp),%eax + rol $30,%esi + add %ebx,%r12d + rol $1,%eax + lea -0x359d3e2a(%eax,%ebp),%r11d + mov %esi,%ebx + mov %r12d,%ebp + xor %edx,%ebx + rol $5,%ebp + xor %edi,%ebx + add %ebp,%r11d + rol $30,%edx + add %ebx,%r11d + // Update and save state information in SHA-1 context + add 0(%r8),%r11d + add 4(%r8),%r12d + add 8(%r8),%edx + add 12(%r8),%esi + add 16(%r8),%edi + mov %r11d,0(%r8) + mov %r12d,4(%r8) + mov %edx,8(%r8) + mov %esi,12(%r8) + mov %edi,16(%r8) + + xchg %r11d,%edx # mov %r11d,%edx + xchg %r12d,%esi # mov %r12d,%esi + xchg %r11d,%edi # mov %edx,%edi + xchg %r12d,%ebp # mov %esi,%ebp + # mov %edi,%r11d + lea 64(%r9),%r9 + sub $1,%r10 + jnz .Lloop + mov 64(%rsp),%rsp + pop %r12 + pop %rbp + pop %rbx + ret +SET_SIZE(sha1_block_data_order) +.asciz "SHA1 block transform for x86_64, CRYPTOGAMS by " + +#endif /* lint || __lint */ diff --git a/module/icp/asm-x86_64/sha2/sha256_impl.S b/module/icp/asm-x86_64/sha2/sha256_impl.S new file mode 100644 index 000000000000..b6a9bbc8634a --- /dev/null +++ b/module/icp/asm-x86_64/sha2/sha256_impl.S @@ -0,0 +1,2060 @@ +/* + * ==================================================================== + * Written by Andy Polyakov for the OpenSSL + * project. Rights for redistribution and usage in source and binary + * forms are granted according to the OpenSSL license. + * ==================================================================== + * + * sha256/512_block procedure for x86_64. + * + * 40% improvement over compiler-generated code on Opteron. On EM64T + * sha256 was observed to run >80% faster and sha512 - >40%. No magical + * tricks, just straight implementation... I really wonder why gcc + * [being armed with inline assembler] fails to generate as fast code. + * The only thing which is cool about this module is that it's very + * same instruction sequence used for both SHA-256 and SHA-512. In + * former case the instructions operate on 32-bit operands, while in + * latter - on 64-bit ones. All I had to do is to get one flavor right, + * the other one passed the test right away:-) + * + * sha256_block runs in ~1005 cycles on Opteron, which gives you + * asymptotic performance of 64*1000/1005=63.7MBps times CPU clock + * frequency in GHz. sha512_block runs in ~1275 cycles, which results + * in 128*1000/1275=100MBps per GHz. Is there room for improvement? + * Well, if you compare it to IA-64 implementation, which maintains + * X[16] in register bank[!], tends to 4 instructions per CPU clock + * cycle and runs in 1003 cycles, 1275 is very good result for 3-way + * issue Opteron pipeline and X[16] maintained in memory. So that *if* + * there is a way to improve it, *then* the only way would be to try to + * offload X[16] updates to SSE unit, but that would require "deeper" + * loop unroll, which in turn would naturally cause size blow-up, not + * to mention increased complexity! And once again, only *if* it's + * actually possible to noticeably improve overall ILP, instruction + * level parallelism, on a given CPU implementation in this case. + * + * Special note on Intel EM64T. While Opteron CPU exhibits perfect + * perfromance ratio of 1.5 between 64- and 32-bit flavors [see above], + * [currently available] EM64T CPUs apparently are far from it. On the + * contrary, 64-bit version, sha512_block, is ~30% *slower* than 32-bit + * sha256_block:-( This is presumably because 64-bit shifts/rotates + * apparently are not atomic instructions, but implemented in microcode. + */ + +/* + * OpenSolaris OS modifications + * + * Sun elects to use this software under the BSD license. + * + * This source originates from OpenSSL file sha512-x86_64.pl at + * ftp://ftp.openssl.org/snapshot/openssl-0.9.8-stable-SNAP-20080131.tar.gz + * (presumably for future OpenSSL release 0.9.8h), with these changes: + * + * 1. Added perl "use strict" and declared variables. + * + * 2. Added OpenSolaris ENTRY_NP/SET_SIZE macros from + * /usr/include/sys/asm_linkage.h, .ident keywords, and lint(1B) guards. + * + * 3. Removed x86_64-xlate.pl script (not needed for as(1) or gas(1) + * assemblers). Replaced the .picmeup macro with assembler code. + * + * 4. Added 8 to $ctx, as OpenSolaris OS has an extra 4-byte field, "algotype", + * at the beginning of SHA2_CTX (the next field is 8-byte aligned). + */ + +/* + * This file was generated by a perl script (sha512-x86_64.pl) that could + * be used to generate sha256 and sha512 variants from the same code base. + * For our purposes, we only need sha256 and so getting the perl script to + * run as part of the build process seemed superfluous. The comments from + * the original file have been pasted above. + */ + +#if defined(lint) || defined(__lint) +#include +#include + +/* ARGSUSED */ +void +SHA256TransformBlocks(SHA2_CTX *ctx, const void *in, size_t num) +{ +} + + +#else +#define _ASM +#include + +ENTRY_NP(SHA256TransformBlocks) + push %rbx + push %rbp + push %r12 + push %r13 + push %r14 + push %r15 + mov %rsp,%rbp # copy %rsp + shl $4,%rdx # num*16 + sub $16*4+4*8,%rsp + lea (%rsi,%rdx,4),%rdx # inp+num*16*4 + and $-64,%rsp # align stack frame + add $8,%rdi # Skip OpenSolaris field, "algotype" + mov %rdi,16*4+0*8(%rsp) # save ctx, 1st arg + mov %rsi,16*4+1*8(%rsp) # save inp, 2nd arg + mov %rdx,16*4+2*8(%rsp) # save end pointer, "3rd" arg + mov %rbp,16*4+3*8(%rsp) # save copy of %rsp + + /.picmeup %rbp + / The .picmeup pseudo-directive, from perlasm/x86_64_xlate.pl, puts + / the address of the "next" instruction into the target register + / (%rbp). This generates these 2 instructions: + lea .Llea(%rip),%rbp + /nop / .picmeup generates a nop for mod 8 alignment--not needed here + +.Llea: + lea K256-.(%rbp),%rbp + + mov 4*0(%rdi),%eax + mov 4*1(%rdi),%ebx + mov 4*2(%rdi),%ecx + mov 4*3(%rdi),%edx + mov 4*4(%rdi),%r8d + mov 4*5(%rdi),%r9d + mov 4*6(%rdi),%r10d + mov 4*7(%rdi),%r11d + jmp .Lloop + +.align 16 +.Lloop: + xor %rdi,%rdi + mov 4*0(%rsi),%r12d + bswap %r12d + mov %r8d,%r13d + mov %r8d,%r14d + mov %r9d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r10d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r8d,%r15d # (f^g)&e + mov %r12d,0(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r10d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r11d,%r12d # T1+=h + + mov %eax,%r11d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %eax,%r13d + mov %eax,%r14d + + ror $2,%r11d + ror $13,%r13d + mov %eax,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r11d + ror $9,%r13d + or %ecx,%r14d # a|c + + xor %r13d,%r11d # h=Sigma0(a) + and %ecx,%r15d # a&c + add %r12d,%edx # d+=T1 + + and %ebx,%r14d # (a|c)&b + add %r12d,%r11d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r11d # h+=Maj(a,b,c) + mov 4*1(%rsi),%r12d + bswap %r12d + mov %edx,%r13d + mov %edx,%r14d + mov %r8d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r9d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %edx,%r15d # (f^g)&e + mov %r12d,4(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r9d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r10d,%r12d # T1+=h + + mov %r11d,%r10d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r11d,%r13d + mov %r11d,%r14d + + ror $2,%r10d + ror $13,%r13d + mov %r11d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r10d + ror $9,%r13d + or %ebx,%r14d # a|c + + xor %r13d,%r10d # h=Sigma0(a) + and %ebx,%r15d # a&c + add %r12d,%ecx # d+=T1 + + and %eax,%r14d # (a|c)&b + add %r12d,%r10d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r10d # h+=Maj(a,b,c) + mov 4*2(%rsi),%r12d + bswap %r12d + mov %ecx,%r13d + mov %ecx,%r14d + mov %edx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r8d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %ecx,%r15d # (f^g)&e + mov %r12d,8(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r8d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r9d,%r12d # T1+=h + + mov %r10d,%r9d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r10d,%r13d + mov %r10d,%r14d + + ror $2,%r9d + ror $13,%r13d + mov %r10d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r9d + ror $9,%r13d + or %eax,%r14d # a|c + + xor %r13d,%r9d # h=Sigma0(a) + and %eax,%r15d # a&c + add %r12d,%ebx # d+=T1 + + and %r11d,%r14d # (a|c)&b + add %r12d,%r9d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r9d # h+=Maj(a,b,c) + mov 4*3(%rsi),%r12d + bswap %r12d + mov %ebx,%r13d + mov %ebx,%r14d + mov %ecx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %edx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %ebx,%r15d # (f^g)&e + mov %r12d,12(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %edx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r8d,%r12d # T1+=h + + mov %r9d,%r8d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r9d,%r13d + mov %r9d,%r14d + + ror $2,%r8d + ror $13,%r13d + mov %r9d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r8d + ror $9,%r13d + or %r11d,%r14d # a|c + + xor %r13d,%r8d # h=Sigma0(a) + and %r11d,%r15d # a&c + add %r12d,%eax # d+=T1 + + and %r10d,%r14d # (a|c)&b + add %r12d,%r8d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r8d # h+=Maj(a,b,c) + mov 4*4(%rsi),%r12d + bswap %r12d + mov %eax,%r13d + mov %eax,%r14d + mov %ebx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %ecx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %eax,%r15d # (f^g)&e + mov %r12d,16(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %ecx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %edx,%r12d # T1+=h + + mov %r8d,%edx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r8d,%r13d + mov %r8d,%r14d + + ror $2,%edx + ror $13,%r13d + mov %r8d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%edx + ror $9,%r13d + or %r10d,%r14d # a|c + + xor %r13d,%edx # h=Sigma0(a) + and %r10d,%r15d # a&c + add %r12d,%r11d # d+=T1 + + and %r9d,%r14d # (a|c)&b + add %r12d,%edx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%edx # h+=Maj(a,b,c) + mov 4*5(%rsi),%r12d + bswap %r12d + mov %r11d,%r13d + mov %r11d,%r14d + mov %eax,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %ebx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r11d,%r15d # (f^g)&e + mov %r12d,20(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %ebx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %ecx,%r12d # T1+=h + + mov %edx,%ecx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %edx,%r13d + mov %edx,%r14d + + ror $2,%ecx + ror $13,%r13d + mov %edx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%ecx + ror $9,%r13d + or %r9d,%r14d # a|c + + xor %r13d,%ecx # h=Sigma0(a) + and %r9d,%r15d # a&c + add %r12d,%r10d # d+=T1 + + and %r8d,%r14d # (a|c)&b + add %r12d,%ecx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%ecx # h+=Maj(a,b,c) + mov 4*6(%rsi),%r12d + bswap %r12d + mov %r10d,%r13d + mov %r10d,%r14d + mov %r11d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %eax,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r10d,%r15d # (f^g)&e + mov %r12d,24(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %eax,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %ebx,%r12d # T1+=h + + mov %ecx,%ebx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %ecx,%r13d + mov %ecx,%r14d + + ror $2,%ebx + ror $13,%r13d + mov %ecx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%ebx + ror $9,%r13d + or %r8d,%r14d # a|c + + xor %r13d,%ebx # h=Sigma0(a) + and %r8d,%r15d # a&c + add %r12d,%r9d # d+=T1 + + and %edx,%r14d # (a|c)&b + add %r12d,%ebx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%ebx # h+=Maj(a,b,c) + mov 4*7(%rsi),%r12d + bswap %r12d + mov %r9d,%r13d + mov %r9d,%r14d + mov %r10d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r11d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r9d,%r15d # (f^g)&e + mov %r12d,28(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r11d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %eax,%r12d # T1+=h + + mov %ebx,%eax + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %ebx,%r13d + mov %ebx,%r14d + + ror $2,%eax + ror $13,%r13d + mov %ebx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%eax + ror $9,%r13d + or %edx,%r14d # a|c + + xor %r13d,%eax # h=Sigma0(a) + and %edx,%r15d # a&c + add %r12d,%r8d # d+=T1 + + and %ecx,%r14d # (a|c)&b + add %r12d,%eax # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%eax # h+=Maj(a,b,c) + mov 4*8(%rsi),%r12d + bswap %r12d + mov %r8d,%r13d + mov %r8d,%r14d + mov %r9d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r10d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r8d,%r15d # (f^g)&e + mov %r12d,32(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r10d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r11d,%r12d # T1+=h + + mov %eax,%r11d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %eax,%r13d + mov %eax,%r14d + + ror $2,%r11d + ror $13,%r13d + mov %eax,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r11d + ror $9,%r13d + or %ecx,%r14d # a|c + + xor %r13d,%r11d # h=Sigma0(a) + and %ecx,%r15d # a&c + add %r12d,%edx # d+=T1 + + and %ebx,%r14d # (a|c)&b + add %r12d,%r11d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r11d # h+=Maj(a,b,c) + mov 4*9(%rsi),%r12d + bswap %r12d + mov %edx,%r13d + mov %edx,%r14d + mov %r8d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r9d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %edx,%r15d # (f^g)&e + mov %r12d,36(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r9d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r10d,%r12d # T1+=h + + mov %r11d,%r10d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r11d,%r13d + mov %r11d,%r14d + + ror $2,%r10d + ror $13,%r13d + mov %r11d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r10d + ror $9,%r13d + or %ebx,%r14d # a|c + + xor %r13d,%r10d # h=Sigma0(a) + and %ebx,%r15d # a&c + add %r12d,%ecx # d+=T1 + + and %eax,%r14d # (a|c)&b + add %r12d,%r10d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r10d # h+=Maj(a,b,c) + mov 4*10(%rsi),%r12d + bswap %r12d + mov %ecx,%r13d + mov %ecx,%r14d + mov %edx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r8d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %ecx,%r15d # (f^g)&e + mov %r12d,40(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r8d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r9d,%r12d # T1+=h + + mov %r10d,%r9d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r10d,%r13d + mov %r10d,%r14d + + ror $2,%r9d + ror $13,%r13d + mov %r10d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r9d + ror $9,%r13d + or %eax,%r14d # a|c + + xor %r13d,%r9d # h=Sigma0(a) + and %eax,%r15d # a&c + add %r12d,%ebx # d+=T1 + + and %r11d,%r14d # (a|c)&b + add %r12d,%r9d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r9d # h+=Maj(a,b,c) + mov 4*11(%rsi),%r12d + bswap %r12d + mov %ebx,%r13d + mov %ebx,%r14d + mov %ecx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %edx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %ebx,%r15d # (f^g)&e + mov %r12d,44(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %edx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r8d,%r12d # T1+=h + + mov %r9d,%r8d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r9d,%r13d + mov %r9d,%r14d + + ror $2,%r8d + ror $13,%r13d + mov %r9d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r8d + ror $9,%r13d + or %r11d,%r14d # a|c + + xor %r13d,%r8d # h=Sigma0(a) + and %r11d,%r15d # a&c + add %r12d,%eax # d+=T1 + + and %r10d,%r14d # (a|c)&b + add %r12d,%r8d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r8d # h+=Maj(a,b,c) + mov 4*12(%rsi),%r12d + bswap %r12d + mov %eax,%r13d + mov %eax,%r14d + mov %ebx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %ecx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %eax,%r15d # (f^g)&e + mov %r12d,48(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %ecx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %edx,%r12d # T1+=h + + mov %r8d,%edx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r8d,%r13d + mov %r8d,%r14d + + ror $2,%edx + ror $13,%r13d + mov %r8d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%edx + ror $9,%r13d + or %r10d,%r14d # a|c + + xor %r13d,%edx # h=Sigma0(a) + and %r10d,%r15d # a&c + add %r12d,%r11d # d+=T1 + + and %r9d,%r14d # (a|c)&b + add %r12d,%edx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%edx # h+=Maj(a,b,c) + mov 4*13(%rsi),%r12d + bswap %r12d + mov %r11d,%r13d + mov %r11d,%r14d + mov %eax,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %ebx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r11d,%r15d # (f^g)&e + mov %r12d,52(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %ebx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %ecx,%r12d # T1+=h + + mov %edx,%ecx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %edx,%r13d + mov %edx,%r14d + + ror $2,%ecx + ror $13,%r13d + mov %edx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%ecx + ror $9,%r13d + or %r9d,%r14d # a|c + + xor %r13d,%ecx # h=Sigma0(a) + and %r9d,%r15d # a&c + add %r12d,%r10d # d+=T1 + + and %r8d,%r14d # (a|c)&b + add %r12d,%ecx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%ecx # h+=Maj(a,b,c) + mov 4*14(%rsi),%r12d + bswap %r12d + mov %r10d,%r13d + mov %r10d,%r14d + mov %r11d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %eax,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r10d,%r15d # (f^g)&e + mov %r12d,56(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %eax,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %ebx,%r12d # T1+=h + + mov %ecx,%ebx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %ecx,%r13d + mov %ecx,%r14d + + ror $2,%ebx + ror $13,%r13d + mov %ecx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%ebx + ror $9,%r13d + or %r8d,%r14d # a|c + + xor %r13d,%ebx # h=Sigma0(a) + and %r8d,%r15d # a&c + add %r12d,%r9d # d+=T1 + + and %edx,%r14d # (a|c)&b + add %r12d,%ebx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%ebx # h+=Maj(a,b,c) + mov 4*15(%rsi),%r12d + bswap %r12d + mov %r9d,%r13d + mov %r9d,%r14d + mov %r10d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r11d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r9d,%r15d # (f^g)&e + mov %r12d,60(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r11d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %eax,%r12d # T1+=h + + mov %ebx,%eax + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %ebx,%r13d + mov %ebx,%r14d + + ror $2,%eax + ror $13,%r13d + mov %ebx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%eax + ror $9,%r13d + or %edx,%r14d # a|c + + xor %r13d,%eax # h=Sigma0(a) + and %edx,%r15d # a&c + add %r12d,%r8d # d+=T1 + + and %ecx,%r14d # (a|c)&b + add %r12d,%eax # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%eax # h+=Maj(a,b,c) + jmp .Lrounds_16_xx +.align 16 +.Lrounds_16_xx: + mov 4(%rsp),%r13d + mov 56(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 36(%rsp),%r12d + + add 0(%rsp),%r12d + mov %r8d,%r13d + mov %r8d,%r14d + mov %r9d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r10d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r8d,%r15d # (f^g)&e + mov %r12d,0(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r10d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r11d,%r12d # T1+=h + + mov %eax,%r11d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %eax,%r13d + mov %eax,%r14d + + ror $2,%r11d + ror $13,%r13d + mov %eax,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r11d + ror $9,%r13d + or %ecx,%r14d # a|c + + xor %r13d,%r11d # h=Sigma0(a) + and %ecx,%r15d # a&c + add %r12d,%edx # d+=T1 + + and %ebx,%r14d # (a|c)&b + add %r12d,%r11d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r11d # h+=Maj(a,b,c) + mov 8(%rsp),%r13d + mov 60(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 40(%rsp),%r12d + + add 4(%rsp),%r12d + mov %edx,%r13d + mov %edx,%r14d + mov %r8d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r9d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %edx,%r15d # (f^g)&e + mov %r12d,4(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r9d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r10d,%r12d # T1+=h + + mov %r11d,%r10d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r11d,%r13d + mov %r11d,%r14d + + ror $2,%r10d + ror $13,%r13d + mov %r11d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r10d + ror $9,%r13d + or %ebx,%r14d # a|c + + xor %r13d,%r10d # h=Sigma0(a) + and %ebx,%r15d # a&c + add %r12d,%ecx # d+=T1 + + and %eax,%r14d # (a|c)&b + add %r12d,%r10d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r10d # h+=Maj(a,b,c) + mov 12(%rsp),%r13d + mov 0(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 44(%rsp),%r12d + + add 8(%rsp),%r12d + mov %ecx,%r13d + mov %ecx,%r14d + mov %edx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r8d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %ecx,%r15d # (f^g)&e + mov %r12d,8(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r8d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r9d,%r12d # T1+=h + + mov %r10d,%r9d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r10d,%r13d + mov %r10d,%r14d + + ror $2,%r9d + ror $13,%r13d + mov %r10d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r9d + ror $9,%r13d + or %eax,%r14d # a|c + + xor %r13d,%r9d # h=Sigma0(a) + and %eax,%r15d # a&c + add %r12d,%ebx # d+=T1 + + and %r11d,%r14d # (a|c)&b + add %r12d,%r9d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r9d # h+=Maj(a,b,c) + mov 16(%rsp),%r13d + mov 4(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 48(%rsp),%r12d + + add 12(%rsp),%r12d + mov %ebx,%r13d + mov %ebx,%r14d + mov %ecx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %edx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %ebx,%r15d # (f^g)&e + mov %r12d,12(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %edx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r8d,%r12d # T1+=h + + mov %r9d,%r8d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r9d,%r13d + mov %r9d,%r14d + + ror $2,%r8d + ror $13,%r13d + mov %r9d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r8d + ror $9,%r13d + or %r11d,%r14d # a|c + + xor %r13d,%r8d # h=Sigma0(a) + and %r11d,%r15d # a&c + add %r12d,%eax # d+=T1 + + and %r10d,%r14d # (a|c)&b + add %r12d,%r8d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r8d # h+=Maj(a,b,c) + mov 20(%rsp),%r13d + mov 8(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 52(%rsp),%r12d + + add 16(%rsp),%r12d + mov %eax,%r13d + mov %eax,%r14d + mov %ebx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %ecx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %eax,%r15d # (f^g)&e + mov %r12d,16(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %ecx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %edx,%r12d # T1+=h + + mov %r8d,%edx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r8d,%r13d + mov %r8d,%r14d + + ror $2,%edx + ror $13,%r13d + mov %r8d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%edx + ror $9,%r13d + or %r10d,%r14d # a|c + + xor %r13d,%edx # h=Sigma0(a) + and %r10d,%r15d # a&c + add %r12d,%r11d # d+=T1 + + and %r9d,%r14d # (a|c)&b + add %r12d,%edx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%edx # h+=Maj(a,b,c) + mov 24(%rsp),%r13d + mov 12(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 56(%rsp),%r12d + + add 20(%rsp),%r12d + mov %r11d,%r13d + mov %r11d,%r14d + mov %eax,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %ebx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r11d,%r15d # (f^g)&e + mov %r12d,20(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %ebx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %ecx,%r12d # T1+=h + + mov %edx,%ecx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %edx,%r13d + mov %edx,%r14d + + ror $2,%ecx + ror $13,%r13d + mov %edx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%ecx + ror $9,%r13d + or %r9d,%r14d # a|c + + xor %r13d,%ecx # h=Sigma0(a) + and %r9d,%r15d # a&c + add %r12d,%r10d # d+=T1 + + and %r8d,%r14d # (a|c)&b + add %r12d,%ecx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%ecx # h+=Maj(a,b,c) + mov 28(%rsp),%r13d + mov 16(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 60(%rsp),%r12d + + add 24(%rsp),%r12d + mov %r10d,%r13d + mov %r10d,%r14d + mov %r11d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %eax,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r10d,%r15d # (f^g)&e + mov %r12d,24(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %eax,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %ebx,%r12d # T1+=h + + mov %ecx,%ebx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %ecx,%r13d + mov %ecx,%r14d + + ror $2,%ebx + ror $13,%r13d + mov %ecx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%ebx + ror $9,%r13d + or %r8d,%r14d # a|c + + xor %r13d,%ebx # h=Sigma0(a) + and %r8d,%r15d # a&c + add %r12d,%r9d # d+=T1 + + and %edx,%r14d # (a|c)&b + add %r12d,%ebx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%ebx # h+=Maj(a,b,c) + mov 32(%rsp),%r13d + mov 20(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 0(%rsp),%r12d + + add 28(%rsp),%r12d + mov %r9d,%r13d + mov %r9d,%r14d + mov %r10d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r11d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r9d,%r15d # (f^g)&e + mov %r12d,28(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r11d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %eax,%r12d # T1+=h + + mov %ebx,%eax + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %ebx,%r13d + mov %ebx,%r14d + + ror $2,%eax + ror $13,%r13d + mov %ebx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%eax + ror $9,%r13d + or %edx,%r14d # a|c + + xor %r13d,%eax # h=Sigma0(a) + and %edx,%r15d # a&c + add %r12d,%r8d # d+=T1 + + and %ecx,%r14d # (a|c)&b + add %r12d,%eax # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%eax # h+=Maj(a,b,c) + mov 36(%rsp),%r13d + mov 24(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 4(%rsp),%r12d + + add 32(%rsp),%r12d + mov %r8d,%r13d + mov %r8d,%r14d + mov %r9d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r10d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r8d,%r15d # (f^g)&e + mov %r12d,32(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r10d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r11d,%r12d # T1+=h + + mov %eax,%r11d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %eax,%r13d + mov %eax,%r14d + + ror $2,%r11d + ror $13,%r13d + mov %eax,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r11d + ror $9,%r13d + or %ecx,%r14d # a|c + + xor %r13d,%r11d # h=Sigma0(a) + and %ecx,%r15d # a&c + add %r12d,%edx # d+=T1 + + and %ebx,%r14d # (a|c)&b + add %r12d,%r11d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r11d # h+=Maj(a,b,c) + mov 40(%rsp),%r13d + mov 28(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 8(%rsp),%r12d + + add 36(%rsp),%r12d + mov %edx,%r13d + mov %edx,%r14d + mov %r8d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r9d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %edx,%r15d # (f^g)&e + mov %r12d,36(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r9d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r10d,%r12d # T1+=h + + mov %r11d,%r10d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r11d,%r13d + mov %r11d,%r14d + + ror $2,%r10d + ror $13,%r13d + mov %r11d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r10d + ror $9,%r13d + or %ebx,%r14d # a|c + + xor %r13d,%r10d # h=Sigma0(a) + and %ebx,%r15d # a&c + add %r12d,%ecx # d+=T1 + + and %eax,%r14d # (a|c)&b + add %r12d,%r10d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r10d # h+=Maj(a,b,c) + mov 44(%rsp),%r13d + mov 32(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 12(%rsp),%r12d + + add 40(%rsp),%r12d + mov %ecx,%r13d + mov %ecx,%r14d + mov %edx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r8d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %ecx,%r15d # (f^g)&e + mov %r12d,40(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r8d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r9d,%r12d # T1+=h + + mov %r10d,%r9d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r10d,%r13d + mov %r10d,%r14d + + ror $2,%r9d + ror $13,%r13d + mov %r10d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r9d + ror $9,%r13d + or %eax,%r14d # a|c + + xor %r13d,%r9d # h=Sigma0(a) + and %eax,%r15d # a&c + add %r12d,%ebx # d+=T1 + + and %r11d,%r14d # (a|c)&b + add %r12d,%r9d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r9d # h+=Maj(a,b,c) + mov 48(%rsp),%r13d + mov 36(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 16(%rsp),%r12d + + add 44(%rsp),%r12d + mov %ebx,%r13d + mov %ebx,%r14d + mov %ecx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %edx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %ebx,%r15d # (f^g)&e + mov %r12d,44(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %edx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %r8d,%r12d # T1+=h + + mov %r9d,%r8d + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r9d,%r13d + mov %r9d,%r14d + + ror $2,%r8d + ror $13,%r13d + mov %r9d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%r8d + ror $9,%r13d + or %r11d,%r14d # a|c + + xor %r13d,%r8d # h=Sigma0(a) + and %r11d,%r15d # a&c + add %r12d,%eax # d+=T1 + + and %r10d,%r14d # (a|c)&b + add %r12d,%r8d # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%r8d # h+=Maj(a,b,c) + mov 52(%rsp),%r13d + mov 40(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 20(%rsp),%r12d + + add 48(%rsp),%r12d + mov %eax,%r13d + mov %eax,%r14d + mov %ebx,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %ecx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %eax,%r15d # (f^g)&e + mov %r12d,48(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %ecx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %edx,%r12d # T1+=h + + mov %r8d,%edx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %r8d,%r13d + mov %r8d,%r14d + + ror $2,%edx + ror $13,%r13d + mov %r8d,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%edx + ror $9,%r13d + or %r10d,%r14d # a|c + + xor %r13d,%edx # h=Sigma0(a) + and %r10d,%r15d # a&c + add %r12d,%r11d # d+=T1 + + and %r9d,%r14d # (a|c)&b + add %r12d,%edx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%edx # h+=Maj(a,b,c) + mov 56(%rsp),%r13d + mov 44(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 24(%rsp),%r12d + + add 52(%rsp),%r12d + mov %r11d,%r13d + mov %r11d,%r14d + mov %eax,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %ebx,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r11d,%r15d # (f^g)&e + mov %r12d,52(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %ebx,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %ecx,%r12d # T1+=h + + mov %edx,%ecx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %edx,%r13d + mov %edx,%r14d + + ror $2,%ecx + ror $13,%r13d + mov %edx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%ecx + ror $9,%r13d + or %r9d,%r14d # a|c + + xor %r13d,%ecx # h=Sigma0(a) + and %r9d,%r15d # a&c + add %r12d,%r10d # d+=T1 + + and %r8d,%r14d # (a|c)&b + add %r12d,%ecx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%ecx # h+=Maj(a,b,c) + mov 60(%rsp),%r13d + mov 48(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 28(%rsp),%r12d + + add 56(%rsp),%r12d + mov %r10d,%r13d + mov %r10d,%r14d + mov %r11d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %eax,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r10d,%r15d # (f^g)&e + mov %r12d,56(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %eax,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %ebx,%r12d # T1+=h + + mov %ecx,%ebx + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %ecx,%r13d + mov %ecx,%r14d + + ror $2,%ebx + ror $13,%r13d + mov %ecx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%ebx + ror $9,%r13d + or %r8d,%r14d # a|c + + xor %r13d,%ebx # h=Sigma0(a) + and %r8d,%r15d # a&c + add %r12d,%r9d # d+=T1 + + and %edx,%r14d # (a|c)&b + add %r12d,%ebx # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%ebx # h+=Maj(a,b,c) + mov 0(%rsp),%r13d + mov 52(%rsp),%r12d + + mov %r13d,%r15d + + shr $3,%r13d + ror $7,%r15d + + xor %r15d,%r13d + ror $11,%r15d + + xor %r15d,%r13d # sigma0(X[(i+1)&0xf]) + mov %r12d,%r14d + + shr $10,%r12d + ror $17,%r14d + + xor %r14d,%r12d + ror $2,%r14d + + xor %r14d,%r12d # sigma1(X[(i+14)&0xf]) + + add %r13d,%r12d + + add 32(%rsp),%r12d + + add 60(%rsp),%r12d + mov %r9d,%r13d + mov %r9d,%r14d + mov %r10d,%r15d + + ror $6,%r13d + ror $11,%r14d + xor %r11d,%r15d # f^g + + xor %r14d,%r13d + ror $14,%r14d + and %r9d,%r15d # (f^g)&e + mov %r12d,60(%rsp) + + xor %r14d,%r13d # Sigma1(e) + xor %r11d,%r15d # Ch(e,f,g)=((f^g)&e)^g + add %eax,%r12d # T1+=h + + mov %ebx,%eax + add %r13d,%r12d # T1+=Sigma1(e) + + add %r15d,%r12d # T1+=Ch(e,f,g) + mov %ebx,%r13d + mov %ebx,%r14d + + ror $2,%eax + ror $13,%r13d + mov %ebx,%r15d + add (%rbp,%rdi,4),%r12d # T1+=K[round] + + xor %r13d,%eax + ror $9,%r13d + or %edx,%r14d # a|c + + xor %r13d,%eax # h=Sigma0(a) + and %edx,%r15d # a&c + add %r12d,%r8d # d+=T1 + + and %ecx,%r14d # (a|c)&b + add %r12d,%eax # h+=T1 + + or %r15d,%r14d # Maj(a,b,c)=((a|c)&b)|(a&c) + lea 1(%rdi),%rdi # round++ + + add %r14d,%eax # h+=Maj(a,b,c) + cmp $64,%rdi + jb .Lrounds_16_xx + + mov 16*4+0*8(%rsp),%rdi + lea 16*4(%rsi),%rsi + + add 4*0(%rdi),%eax + add 4*1(%rdi),%ebx + add 4*2(%rdi),%ecx + add 4*3(%rdi),%edx + add 4*4(%rdi),%r8d + add 4*5(%rdi),%r9d + add 4*6(%rdi),%r10d + add 4*7(%rdi),%r11d + + cmp 16*4+2*8(%rsp),%rsi + + mov %eax,4*0(%rdi) + mov %ebx,4*1(%rdi) + mov %ecx,4*2(%rdi) + mov %edx,4*3(%rdi) + mov %r8d,4*4(%rdi) + mov %r9d,4*5(%rdi) + mov %r10d,4*6(%rdi) + mov %r11d,4*7(%rdi) + jb .Lloop + + mov 16*4+3*8(%rsp),%rsp + pop %r15 + pop %r14 + pop %r13 + pop %r12 + pop %rbp + pop %rbx + + ret +SET_SIZE(SHA256TransformBlocks) + +.align 64 +.type K256,@object +K256: + .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5 + .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5 + .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3 + .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174 + .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc + .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da + .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7 + .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967 + .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13 + .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85 + .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3 + .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070 + .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5 + .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3 + .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208 + .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2 +#endif /* !lint && !__lint */ diff --git a/module/icp/core/kcf_callprov.c b/module/icp/core/kcf_callprov.c new file mode 100644 index 000000000000..38927dcc050a --- /dev/null +++ b/module/icp/core/kcf_callprov.c @@ -0,0 +1,1567 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include + +static int kcf_emulate_dual(kcf_provider_desc_t *, crypto_ctx_t *, + kcf_req_params_t *); + +void +kcf_free_triedlist(kcf_prov_tried_t *list) +{ + kcf_prov_tried_t *l; + + while ((l = list) != NULL) { + list = list->pt_next; + KCF_PROV_REFRELE(l->pt_pd); + kmem_free(l, sizeof (kcf_prov_tried_t)); + } +} + +kcf_prov_tried_t * +kcf_insert_triedlist(kcf_prov_tried_t **list, kcf_provider_desc_t *pd, + int kmflag) +{ + kcf_prov_tried_t *l; + + l = kmem_alloc(sizeof (kcf_prov_tried_t), kmflag); + if (l == NULL) + return (NULL); + + l->pt_pd = pd; + l->pt_next = *list; + *list = l; + + return (l); +} + +static boolean_t +is_in_triedlist(kcf_provider_desc_t *pd, kcf_prov_tried_t *triedl) +{ + while (triedl != NULL) { + if (triedl->pt_pd == pd) + return (B_TRUE); + triedl = triedl->pt_next; + }; + + return (B_FALSE); +} + +/* + * Search a mech entry's hardware provider list for the specified + * provider. Return true if found. + */ +static boolean_t +is_valid_provider_for_mech(kcf_provider_desc_t *pd, kcf_mech_entry_t *me, + crypto_func_group_t fg) +{ + kcf_prov_mech_desc_t *prov_chain; + + prov_chain = me->me_hw_prov_chain; + if (prov_chain != NULL) { + ASSERT(me->me_num_hwprov > 0); + for (; prov_chain != NULL; prov_chain = prov_chain->pm_next) { + if (prov_chain->pm_prov_desc == pd && + IS_FG_SUPPORTED(prov_chain, fg)) { + return (B_TRUE); + } + } + } + return (B_FALSE); +} + +/* + * This routine, given a logical provider, returns the least loaded + * provider belonging to the logical provider. The provider must be + * able to do the specified mechanism, i.e. check that the mechanism + * hasn't been disabled. In addition, just in case providers are not + * entirely equivalent, the provider's entry point is checked for + * non-nullness. This is accomplished by having the caller pass, as + * arguments, the offset of the function group (offset_1), and the + * offset of the function within the function group (offset_2). + * Returns NULL if no provider can be found. + */ +int +kcf_get_hardware_provider(crypto_mech_type_t mech_type_1, + crypto_mech_type_t mech_type_2, boolean_t call_restrict, + kcf_provider_desc_t *old, kcf_provider_desc_t **new, crypto_func_group_t fg) +{ + kcf_provider_desc_t *provider, *real_pd = old; + kcf_provider_desc_t *gpd = NULL; /* good provider */ + kcf_provider_desc_t *bpd = NULL; /* busy provider */ + kcf_provider_list_t *p; + kcf_ops_class_t class; + kcf_mech_entry_t *me; + kcf_mech_entry_tab_t *me_tab; + int index, len, gqlen = INT_MAX, rv = CRYPTO_SUCCESS; + + /* get the mech entry for the specified mechanism */ + class = KCF_MECH2CLASS(mech_type_1); + if ((class < KCF_FIRST_OPSCLASS) || (class > KCF_LAST_OPSCLASS)) { + return (CRYPTO_MECHANISM_INVALID); + } + + me_tab = &kcf_mech_tabs_tab[class]; + index = KCF_MECH2INDEX(mech_type_1); + if ((index < 0) || (index >= me_tab->met_size)) { + return (CRYPTO_MECHANISM_INVALID); + } + + me = &((me_tab->met_tab)[index]); + mutex_enter(&me->me_mutex); + + /* + * We assume the provider descriptor will not go away because + * it is being held somewhere, i.e. its reference count has been + * incremented. In the case of the crypto module, the provider + * descriptor is held by the session structure. + */ + if (old->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + if (old->pd_provider_list == NULL) { + real_pd = NULL; + rv = CRYPTO_DEVICE_ERROR; + goto out; + } + /* + * Find the least loaded real provider. KCF_PROV_LOAD gives + * the load (number of pending requests) of the provider. + */ + mutex_enter(&old->pd_lock); + p = old->pd_provider_list; + while (p != NULL) { + provider = p->pl_provider; + + ASSERT(provider->pd_prov_type != + CRYPTO_LOGICAL_PROVIDER); + + if (call_restrict && + (provider->pd_flags & KCF_PROV_RESTRICTED)) { + p = p->pl_next; + continue; + } + + if (!is_valid_provider_for_mech(provider, me, fg)) { + p = p->pl_next; + continue; + } + + /* provider does second mech */ + if (mech_type_2 != CRYPTO_MECH_INVALID) { + int i; + + i = KCF_TO_PROV_MECH_INDX(provider, + mech_type_2); + if (i == KCF_INVALID_INDX) { + p = p->pl_next; + continue; + } + } + + if (provider->pd_state != KCF_PROV_READY) { + /* choose BUSY if no READY providers */ + if (provider->pd_state == KCF_PROV_BUSY) + bpd = provider; + p = p->pl_next; + continue; + } + + len = KCF_PROV_LOAD(provider); + if (len < gqlen) { + gqlen = len; + gpd = provider; + } + + p = p->pl_next; + } + + if (gpd != NULL) { + real_pd = gpd; + KCF_PROV_REFHOLD(real_pd); + } else if (bpd != NULL) { + real_pd = bpd; + KCF_PROV_REFHOLD(real_pd); + } else { + /* can't find provider */ + real_pd = NULL; + rv = CRYPTO_MECHANISM_INVALID; + } + mutex_exit(&old->pd_lock); + + } else { + if (!KCF_IS_PROV_USABLE(old) || + (call_restrict && (old->pd_flags & KCF_PROV_RESTRICTED))) { + real_pd = NULL; + rv = CRYPTO_DEVICE_ERROR; + goto out; + } + + if (!is_valid_provider_for_mech(old, me, fg)) { + real_pd = NULL; + rv = CRYPTO_MECHANISM_INVALID; + goto out; + } + + KCF_PROV_REFHOLD(real_pd); + } +out: + mutex_exit(&me->me_mutex); + *new = real_pd; + return (rv); +} + +/* + * Return the best provider for the specified mechanism. The provider + * is held and it is the caller's responsibility to release it when done. + * The fg input argument is used as a search criterion to pick a provider. + * A provider has to support this function group to be picked. + * + * Find the least loaded provider in the list of providers. We do a linear + * search to find one. This is fine as we assume there are only a few + * number of providers in this list. If this assumption ever changes, + * we should revisit this. + * + * call_restrict represents if the caller should not be allowed to + * use restricted providers. + */ +kcf_provider_desc_t * +kcf_get_mech_provider(crypto_mech_type_t mech_type, kcf_mech_entry_t **mepp, + int *error, kcf_prov_tried_t *triedl, crypto_func_group_t fg, + boolean_t call_restrict, size_t data_size) +{ + kcf_provider_desc_t *pd = NULL, *gpd = NULL; + kcf_prov_mech_desc_t *prov_chain, *mdesc; + int len, gqlen = INT_MAX; + kcf_ops_class_t class; + int index; + kcf_mech_entry_t *me; + kcf_mech_entry_tab_t *me_tab; + + class = KCF_MECH2CLASS(mech_type); + if ((class < KCF_FIRST_OPSCLASS) || (class > KCF_LAST_OPSCLASS)) { + *error = CRYPTO_MECHANISM_INVALID; + return (NULL); + } + + me_tab = &kcf_mech_tabs_tab[class]; + index = KCF_MECH2INDEX(mech_type); + if ((index < 0) || (index >= me_tab->met_size)) { + *error = CRYPTO_MECHANISM_INVALID; + return (NULL); + } + + me = &((me_tab->met_tab)[index]); + if (mepp != NULL) + *mepp = me; + + mutex_enter(&me->me_mutex); + + prov_chain = me->me_hw_prov_chain; + + /* + * We check for the threshhold for using a hardware provider for + * this amount of data. If there is no software provider available + * for the mechanism, then the threshold is ignored. + */ + if ((prov_chain != NULL) && + ((data_size == 0) || (me->me_threshold == 0) || + (data_size >= me->me_threshold) || + ((mdesc = me->me_sw_prov) == NULL) || + (!IS_FG_SUPPORTED(mdesc, fg)) || + (!KCF_IS_PROV_USABLE(mdesc->pm_prov_desc)))) { + ASSERT(me->me_num_hwprov > 0); + /* there is at least one provider */ + + /* + * Find the least loaded real provider. KCF_PROV_LOAD gives + * the load (number of pending requests) of the provider. + */ + while (prov_chain != NULL) { + pd = prov_chain->pm_prov_desc; + + if (!IS_FG_SUPPORTED(prov_chain, fg) || + !KCF_IS_PROV_USABLE(pd) || + IS_PROVIDER_TRIED(pd, triedl) || + (call_restrict && + (pd->pd_flags & KCF_PROV_RESTRICTED))) { + prov_chain = prov_chain->pm_next; + continue; + } + + if ((len = KCF_PROV_LOAD(pd)) < gqlen) { + gqlen = len; + gpd = pd; + } + + prov_chain = prov_chain->pm_next; + } + + pd = gpd; + } + + /* No HW provider for this mech, is there a SW provider? */ + if (pd == NULL && (mdesc = me->me_sw_prov) != NULL) { + pd = mdesc->pm_prov_desc; + if (!IS_FG_SUPPORTED(mdesc, fg) || + !KCF_IS_PROV_USABLE(pd) || + IS_PROVIDER_TRIED(pd, triedl) || + (call_restrict && (pd->pd_flags & KCF_PROV_RESTRICTED))) + pd = NULL; + } + + if (pd == NULL) { + /* + * We do not want to report CRYPTO_MECH_NOT_SUPPORTED, when + * we are in the "fallback to the next provider" case. Rather + * we preserve the error, so that the client gets the right + * error code. + */ + if (triedl == NULL) + *error = CRYPTO_MECH_NOT_SUPPORTED; + } else + KCF_PROV_REFHOLD(pd); + + mutex_exit(&me->me_mutex); + return (pd); +} + +/* + * Very similar to kcf_get_mech_provider(). Finds the best provider capable of + * a dual operation with both me1 and me2. + * When no dual-ops capable providers are available, return the best provider + * for me1 only, and sets *prov_mt2 to CRYPTO_INVALID_MECHID; + * We assume/expect that a slower HW capable of the dual is still + * faster than the 2 fastest providers capable of the individual ops + * separately. + */ +kcf_provider_desc_t * +kcf_get_dual_provider(crypto_mechanism_t *mech1, crypto_mechanism_t *mech2, + kcf_mech_entry_t **mepp, crypto_mech_type_t *prov_mt1, + crypto_mech_type_t *prov_mt2, int *error, kcf_prov_tried_t *triedl, + crypto_func_group_t fg1, crypto_func_group_t fg2, boolean_t call_restrict, + size_t data_size) +{ + kcf_provider_desc_t *pd = NULL, *pdm1 = NULL, *pdm1m2 = NULL; + kcf_prov_mech_desc_t *prov_chain, *mdesc; + int len, gqlen = INT_MAX, dgqlen = INT_MAX; + crypto_mech_info_list_t *mil; + crypto_mech_type_t m2id = mech2->cm_type; + kcf_mech_entry_t *me; + + /* when mech is a valid mechanism, me will be its mech_entry */ + if (kcf_get_mech_entry(mech1->cm_type, &me) != KCF_SUCCESS) { + *error = CRYPTO_MECHANISM_INVALID; + return (NULL); + } + + *prov_mt2 = CRYPTO_MECH_INVALID; + + if (mepp != NULL) + *mepp = me; + mutex_enter(&me->me_mutex); + + prov_chain = me->me_hw_prov_chain; + /* + * We check the threshold for using a hardware provider for + * this amount of data. If there is no software provider available + * for the first mechanism, then the threshold is ignored. + */ + if ((prov_chain != NULL) && + ((data_size == 0) || (me->me_threshold == 0) || + (data_size >= me->me_threshold) || + ((mdesc = me->me_sw_prov) == NULL) || + (!IS_FG_SUPPORTED(mdesc, fg1)) || + (!KCF_IS_PROV_USABLE(mdesc->pm_prov_desc)))) { + /* there is at least one provider */ + ASSERT(me->me_num_hwprov > 0); + + /* + * Find the least loaded provider capable of the combo + * me1 + me2, and save a pointer to the least loaded + * provider capable of me1 only. + */ + while (prov_chain != NULL) { + pd = prov_chain->pm_prov_desc; + len = KCF_PROV_LOAD(pd); + + if (!IS_FG_SUPPORTED(prov_chain, fg1) || + !KCF_IS_PROV_USABLE(pd) || + IS_PROVIDER_TRIED(pd, triedl) || + (call_restrict && + (pd->pd_flags & KCF_PROV_RESTRICTED))) { + prov_chain = prov_chain->pm_next; + continue; + } + + /* Save the best provider capable of m1 */ + if (len < gqlen) { + *prov_mt1 = + prov_chain->pm_mech_info.cm_mech_number; + gqlen = len; + pdm1 = pd; + } + + /* See if pd can do me2 too */ + for (mil = prov_chain->pm_mi_list; + mil != NULL; mil = mil->ml_next) { + if ((mil->ml_mech_info.cm_func_group_mask & + fg2) == 0) + continue; + + if ((mil->ml_kcf_mechid == m2id) && + (len < dgqlen)) { + /* Bingo! */ + dgqlen = len; + pdm1m2 = pd; + *prov_mt2 = + mil->ml_mech_info.cm_mech_number; + *prov_mt1 = prov_chain-> + pm_mech_info.cm_mech_number; + break; + } + } + + prov_chain = prov_chain->pm_next; + } + + pd = (pdm1m2 != NULL) ? pdm1m2 : pdm1; + } + + /* no HW provider for this mech, is there a SW provider? */ + if (pd == NULL && (mdesc = me->me_sw_prov) != NULL) { + pd = mdesc->pm_prov_desc; + if (!IS_FG_SUPPORTED(mdesc, fg1) || + !KCF_IS_PROV_USABLE(pd) || + IS_PROVIDER_TRIED(pd, triedl) || + (call_restrict && (pd->pd_flags & KCF_PROV_RESTRICTED))) + pd = NULL; + else { + /* See if pd can do me2 too */ + for (mil = me->me_sw_prov->pm_mi_list; + mil != NULL; mil = mil->ml_next) { + if ((mil->ml_mech_info.cm_func_group_mask & + fg2) == 0) + continue; + + if (mil->ml_kcf_mechid == m2id) { + /* Bingo! */ + *prov_mt2 = + mil->ml_mech_info.cm_mech_number; + break; + } + } + *prov_mt1 = me->me_sw_prov->pm_mech_info.cm_mech_number; + } + } + + if (pd == NULL) + *error = CRYPTO_MECH_NOT_SUPPORTED; + else + KCF_PROV_REFHOLD(pd); + + mutex_exit(&me->me_mutex); + return (pd); +} + +/* + * Do the actual work of calling the provider routines. + * + * pd - Provider structure + * ctx - Context for this operation + * params - Parameters for this operation + * rhndl - Request handle to use for notification + * + * The return values are the same as that of the respective SPI. + */ +int +common_submit_request(kcf_provider_desc_t *pd, crypto_ctx_t *ctx, + kcf_req_params_t *params, crypto_req_handle_t rhndl) +{ + int err = CRYPTO_ARGUMENTS_BAD; + kcf_op_type_t optype; + + optype = params->rp_optype; + + switch (params->rp_opgrp) { + case KCF_OG_DIGEST: { + kcf_digest_ops_params_t *dops = ¶ms->rp_u.digest_params; + + switch (optype) { + case KCF_OP_INIT: + /* + * We should do this only here and not in KCF_WRAP_* + * macros. This is because we may want to try other + * providers, in case we recover from a failure. + */ + KCF_SET_PROVIDER_MECHNUM(dops->do_framework_mechtype, + pd, &dops->do_mech); + + err = KCF_PROV_DIGEST_INIT(pd, ctx, &dops->do_mech, + rhndl); + break; + + case KCF_OP_SINGLE: + err = KCF_PROV_DIGEST(pd, ctx, dops->do_data, + dops->do_digest, rhndl); + break; + + case KCF_OP_UPDATE: + err = KCF_PROV_DIGEST_UPDATE(pd, ctx, + dops->do_data, rhndl); + break; + + case KCF_OP_FINAL: + err = KCF_PROV_DIGEST_FINAL(pd, ctx, + dops->do_digest, rhndl); + break; + + case KCF_OP_ATOMIC: + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(dops->do_framework_mechtype, + pd, &dops->do_mech); + err = KCF_PROV_DIGEST_ATOMIC(pd, dops->do_sid, + &dops->do_mech, dops->do_data, dops->do_digest, + rhndl); + break; + + case KCF_OP_DIGEST_KEY: + err = KCF_PROV_DIGEST_KEY(pd, ctx, dops->do_digest_key, + rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_MAC: { + kcf_mac_ops_params_t *mops = ¶ms->rp_u.mac_params; + + switch (optype) { + case KCF_OP_INIT: + KCF_SET_PROVIDER_MECHNUM(mops->mo_framework_mechtype, + pd, &mops->mo_mech); + + err = KCF_PROV_MAC_INIT(pd, ctx, &mops->mo_mech, + mops->mo_key, mops->mo_templ, rhndl); + break; + + case KCF_OP_SINGLE: + err = KCF_PROV_MAC(pd, ctx, mops->mo_data, + mops->mo_mac, rhndl); + break; + + case KCF_OP_UPDATE: + err = KCF_PROV_MAC_UPDATE(pd, ctx, mops->mo_data, + rhndl); + break; + + case KCF_OP_FINAL: + err = KCF_PROV_MAC_FINAL(pd, ctx, mops->mo_mac, rhndl); + break; + + case KCF_OP_ATOMIC: + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(mops->mo_framework_mechtype, + pd, &mops->mo_mech); + + err = KCF_PROV_MAC_ATOMIC(pd, mops->mo_sid, + &mops->mo_mech, mops->mo_key, mops->mo_data, + mops->mo_mac, mops->mo_templ, rhndl); + break; + + case KCF_OP_MAC_VERIFY_ATOMIC: + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(mops->mo_framework_mechtype, + pd, &mops->mo_mech); + + err = KCF_PROV_MAC_VERIFY_ATOMIC(pd, mops->mo_sid, + &mops->mo_mech, mops->mo_key, mops->mo_data, + mops->mo_mac, mops->mo_templ, rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_ENCRYPT: { + kcf_encrypt_ops_params_t *eops = ¶ms->rp_u.encrypt_params; + + switch (optype) { + case KCF_OP_INIT: + KCF_SET_PROVIDER_MECHNUM(eops->eo_framework_mechtype, + pd, &eops->eo_mech); + + err = KCF_PROV_ENCRYPT_INIT(pd, ctx, &eops->eo_mech, + eops->eo_key, eops->eo_templ, rhndl); + break; + + case KCF_OP_SINGLE: + err = KCF_PROV_ENCRYPT(pd, ctx, eops->eo_plaintext, + eops->eo_ciphertext, rhndl); + break; + + case KCF_OP_UPDATE: + err = KCF_PROV_ENCRYPT_UPDATE(pd, ctx, + eops->eo_plaintext, eops->eo_ciphertext, rhndl); + break; + + case KCF_OP_FINAL: + err = KCF_PROV_ENCRYPT_FINAL(pd, ctx, + eops->eo_ciphertext, rhndl); + break; + + case KCF_OP_ATOMIC: + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(eops->eo_framework_mechtype, + pd, &eops->eo_mech); + + err = KCF_PROV_ENCRYPT_ATOMIC(pd, eops->eo_sid, + &eops->eo_mech, eops->eo_key, eops->eo_plaintext, + eops->eo_ciphertext, eops->eo_templ, rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_DECRYPT: { + kcf_decrypt_ops_params_t *dcrops = ¶ms->rp_u.decrypt_params; + + switch (optype) { + case KCF_OP_INIT: + KCF_SET_PROVIDER_MECHNUM(dcrops->dop_framework_mechtype, + pd, &dcrops->dop_mech); + + err = KCF_PROV_DECRYPT_INIT(pd, ctx, &dcrops->dop_mech, + dcrops->dop_key, dcrops->dop_templ, rhndl); + break; + + case KCF_OP_SINGLE: + err = KCF_PROV_DECRYPT(pd, ctx, dcrops->dop_ciphertext, + dcrops->dop_plaintext, rhndl); + break; + + case KCF_OP_UPDATE: + err = KCF_PROV_DECRYPT_UPDATE(pd, ctx, + dcrops->dop_ciphertext, dcrops->dop_plaintext, + rhndl); + break; + + case KCF_OP_FINAL: + err = KCF_PROV_DECRYPT_FINAL(pd, ctx, + dcrops->dop_plaintext, rhndl); + break; + + case KCF_OP_ATOMIC: + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(dcrops->dop_framework_mechtype, + pd, &dcrops->dop_mech); + + err = KCF_PROV_DECRYPT_ATOMIC(pd, dcrops->dop_sid, + &dcrops->dop_mech, dcrops->dop_key, + dcrops->dop_ciphertext, dcrops->dop_plaintext, + dcrops->dop_templ, rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_SIGN: { + kcf_sign_ops_params_t *sops = ¶ms->rp_u.sign_params; + + switch (optype) { + case KCF_OP_INIT: + KCF_SET_PROVIDER_MECHNUM(sops->so_framework_mechtype, + pd, &sops->so_mech); + + err = KCF_PROV_SIGN_INIT(pd, ctx, &sops->so_mech, + sops->so_key, sops->so_templ, rhndl); + break; + + case KCF_OP_SIGN_RECOVER_INIT: + KCF_SET_PROVIDER_MECHNUM(sops->so_framework_mechtype, + pd, &sops->so_mech); + + err = KCF_PROV_SIGN_RECOVER_INIT(pd, ctx, + &sops->so_mech, sops->so_key, sops->so_templ, + rhndl); + break; + + case KCF_OP_SINGLE: + err = KCF_PROV_SIGN(pd, ctx, sops->so_data, + sops->so_signature, rhndl); + break; + + case KCF_OP_SIGN_RECOVER: + err = KCF_PROV_SIGN_RECOVER(pd, ctx, + sops->so_data, sops->so_signature, rhndl); + break; + + case KCF_OP_UPDATE: + err = KCF_PROV_SIGN_UPDATE(pd, ctx, sops->so_data, + rhndl); + break; + + case KCF_OP_FINAL: + err = KCF_PROV_SIGN_FINAL(pd, ctx, sops->so_signature, + rhndl); + break; + + case KCF_OP_ATOMIC: + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(sops->so_framework_mechtype, + pd, &sops->so_mech); + + err = KCF_PROV_SIGN_ATOMIC(pd, sops->so_sid, + &sops->so_mech, sops->so_key, sops->so_data, + sops->so_templ, sops->so_signature, rhndl); + break; + + case KCF_OP_SIGN_RECOVER_ATOMIC: + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(sops->so_framework_mechtype, + pd, &sops->so_mech); + + err = KCF_PROV_SIGN_RECOVER_ATOMIC(pd, sops->so_sid, + &sops->so_mech, sops->so_key, sops->so_data, + sops->so_templ, sops->so_signature, rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_VERIFY: { + kcf_verify_ops_params_t *vops = ¶ms->rp_u.verify_params; + + switch (optype) { + case KCF_OP_INIT: + KCF_SET_PROVIDER_MECHNUM(vops->vo_framework_mechtype, + pd, &vops->vo_mech); + + err = KCF_PROV_VERIFY_INIT(pd, ctx, &vops->vo_mech, + vops->vo_key, vops->vo_templ, rhndl); + break; + + case KCF_OP_VERIFY_RECOVER_INIT: + KCF_SET_PROVIDER_MECHNUM(vops->vo_framework_mechtype, + pd, &vops->vo_mech); + + err = KCF_PROV_VERIFY_RECOVER_INIT(pd, ctx, + &vops->vo_mech, vops->vo_key, vops->vo_templ, + rhndl); + break; + + case KCF_OP_SINGLE: + err = KCF_PROV_VERIFY(pd, ctx, vops->vo_data, + vops->vo_signature, rhndl); + break; + + case KCF_OP_VERIFY_RECOVER: + err = KCF_PROV_VERIFY_RECOVER(pd, ctx, + vops->vo_signature, vops->vo_data, rhndl); + break; + + case KCF_OP_UPDATE: + err = KCF_PROV_VERIFY_UPDATE(pd, ctx, vops->vo_data, + rhndl); + break; + + case KCF_OP_FINAL: + err = KCF_PROV_VERIFY_FINAL(pd, ctx, vops->vo_signature, + rhndl); + break; + + case KCF_OP_ATOMIC: + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(vops->vo_framework_mechtype, + pd, &vops->vo_mech); + + err = KCF_PROV_VERIFY_ATOMIC(pd, vops->vo_sid, + &vops->vo_mech, vops->vo_key, vops->vo_data, + vops->vo_templ, vops->vo_signature, rhndl); + break; + + case KCF_OP_VERIFY_RECOVER_ATOMIC: + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(vops->vo_framework_mechtype, + pd, &vops->vo_mech); + + err = KCF_PROV_VERIFY_RECOVER_ATOMIC(pd, vops->vo_sid, + &vops->vo_mech, vops->vo_key, vops->vo_signature, + vops->vo_templ, vops->vo_data, rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_ENCRYPT_MAC: { + kcf_encrypt_mac_ops_params_t *eops = + ¶ms->rp_u.encrypt_mac_params; + kcf_context_t *kcf_secondctx; + + switch (optype) { + case KCF_OP_INIT: + kcf_secondctx = ((kcf_context_t *) + (ctx->cc_framework_private))->kc_secondctx; + + if (kcf_secondctx != NULL) { + err = kcf_emulate_dual(pd, ctx, params); + break; + } + KCF_SET_PROVIDER_MECHNUM( + eops->em_framework_encr_mechtype, + pd, &eops->em_encr_mech); + + KCF_SET_PROVIDER_MECHNUM( + eops->em_framework_mac_mechtype, + pd, &eops->em_mac_mech); + + err = KCF_PROV_ENCRYPT_MAC_INIT(pd, ctx, + &eops->em_encr_mech, eops->em_encr_key, + &eops->em_mac_mech, eops->em_mac_key, + eops->em_encr_templ, eops->em_mac_templ, + rhndl); + + break; + + case KCF_OP_SINGLE: + err = KCF_PROV_ENCRYPT_MAC(pd, ctx, + eops->em_plaintext, eops->em_ciphertext, + eops->em_mac, rhndl); + break; + + case KCF_OP_UPDATE: + kcf_secondctx = ((kcf_context_t *) + (ctx->cc_framework_private))->kc_secondctx; + if (kcf_secondctx != NULL) { + err = kcf_emulate_dual(pd, ctx, params); + break; + } + err = KCF_PROV_ENCRYPT_MAC_UPDATE(pd, ctx, + eops->em_plaintext, eops->em_ciphertext, rhndl); + break; + + case KCF_OP_FINAL: + kcf_secondctx = ((kcf_context_t *) + (ctx->cc_framework_private))->kc_secondctx; + if (kcf_secondctx != NULL) { + err = kcf_emulate_dual(pd, ctx, params); + break; + } + err = KCF_PROV_ENCRYPT_MAC_FINAL(pd, ctx, + eops->em_ciphertext, eops->em_mac, rhndl); + break; + + case KCF_OP_ATOMIC: + ASSERT(ctx == NULL); + + KCF_SET_PROVIDER_MECHNUM( + eops->em_framework_encr_mechtype, + pd, &eops->em_encr_mech); + + KCF_SET_PROVIDER_MECHNUM( + eops->em_framework_mac_mechtype, + pd, &eops->em_mac_mech); + + err = KCF_PROV_ENCRYPT_MAC_ATOMIC(pd, eops->em_sid, + &eops->em_encr_mech, eops->em_encr_key, + &eops->em_mac_mech, eops->em_mac_key, + eops->em_plaintext, eops->em_ciphertext, + eops->em_mac, + eops->em_encr_templ, eops->em_mac_templ, + rhndl); + + break; + + default: + break; + } + break; + } + + case KCF_OG_MAC_DECRYPT: { + kcf_mac_decrypt_ops_params_t *dops = + ¶ms->rp_u.mac_decrypt_params; + kcf_context_t *kcf_secondctx; + + switch (optype) { + case KCF_OP_INIT: + kcf_secondctx = ((kcf_context_t *) + (ctx->cc_framework_private))->kc_secondctx; + + if (kcf_secondctx != NULL) { + err = kcf_emulate_dual(pd, ctx, params); + break; + } + KCF_SET_PROVIDER_MECHNUM( + dops->md_framework_mac_mechtype, + pd, &dops->md_mac_mech); + + KCF_SET_PROVIDER_MECHNUM( + dops->md_framework_decr_mechtype, + pd, &dops->md_decr_mech); + + err = KCF_PROV_MAC_DECRYPT_INIT(pd, ctx, + &dops->md_mac_mech, dops->md_mac_key, + &dops->md_decr_mech, dops->md_decr_key, + dops->md_mac_templ, dops->md_decr_templ, + rhndl); + + break; + + case KCF_OP_SINGLE: + err = KCF_PROV_MAC_DECRYPT(pd, ctx, + dops->md_ciphertext, dops->md_mac, + dops->md_plaintext, rhndl); + break; + + case KCF_OP_UPDATE: + kcf_secondctx = ((kcf_context_t *) + (ctx->cc_framework_private))->kc_secondctx; + if (kcf_secondctx != NULL) { + err = kcf_emulate_dual(pd, ctx, params); + break; + } + err = KCF_PROV_MAC_DECRYPT_UPDATE(pd, ctx, + dops->md_ciphertext, dops->md_plaintext, rhndl); + break; + + case KCF_OP_FINAL: + kcf_secondctx = ((kcf_context_t *) + (ctx->cc_framework_private))->kc_secondctx; + if (kcf_secondctx != NULL) { + err = kcf_emulate_dual(pd, ctx, params); + break; + } + err = KCF_PROV_MAC_DECRYPT_FINAL(pd, ctx, + dops->md_mac, dops->md_plaintext, rhndl); + break; + + case KCF_OP_ATOMIC: + ASSERT(ctx == NULL); + + KCF_SET_PROVIDER_MECHNUM( + dops->md_framework_mac_mechtype, + pd, &dops->md_mac_mech); + + KCF_SET_PROVIDER_MECHNUM( + dops->md_framework_decr_mechtype, + pd, &dops->md_decr_mech); + + err = KCF_PROV_MAC_DECRYPT_ATOMIC(pd, dops->md_sid, + &dops->md_mac_mech, dops->md_mac_key, + &dops->md_decr_mech, dops->md_decr_key, + dops->md_ciphertext, dops->md_mac, + dops->md_plaintext, + dops->md_mac_templ, dops->md_decr_templ, + rhndl); + + break; + + case KCF_OP_MAC_VERIFY_DECRYPT_ATOMIC: + ASSERT(ctx == NULL); + + KCF_SET_PROVIDER_MECHNUM( + dops->md_framework_mac_mechtype, + pd, &dops->md_mac_mech); + + KCF_SET_PROVIDER_MECHNUM( + dops->md_framework_decr_mechtype, + pd, &dops->md_decr_mech); + + err = KCF_PROV_MAC_VERIFY_DECRYPT_ATOMIC(pd, + dops->md_sid, &dops->md_mac_mech, dops->md_mac_key, + &dops->md_decr_mech, dops->md_decr_key, + dops->md_ciphertext, dops->md_mac, + dops->md_plaintext, + dops->md_mac_templ, dops->md_decr_templ, + rhndl); + + break; + + default: + break; + } + break; + } + + case KCF_OG_KEY: { + kcf_key_ops_params_t *kops = ¶ms->rp_u.key_params; + + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(kops->ko_framework_mechtype, pd, + &kops->ko_mech); + + switch (optype) { + case KCF_OP_KEY_GENERATE: + err = KCF_PROV_KEY_GENERATE(pd, kops->ko_sid, + &kops->ko_mech, + kops->ko_key_template, kops->ko_key_attribute_count, + kops->ko_key_object_id_ptr, rhndl); + break; + + case KCF_OP_KEY_GENERATE_PAIR: + err = KCF_PROV_KEY_GENERATE_PAIR(pd, kops->ko_sid, + &kops->ko_mech, + kops->ko_key_template, kops->ko_key_attribute_count, + kops->ko_private_key_template, + kops->ko_private_key_attribute_count, + kops->ko_key_object_id_ptr, + kops->ko_private_key_object_id_ptr, rhndl); + break; + + case KCF_OP_KEY_WRAP: + err = KCF_PROV_KEY_WRAP(pd, kops->ko_sid, + &kops->ko_mech, + kops->ko_key, kops->ko_key_object_id_ptr, + kops->ko_wrapped_key, kops->ko_wrapped_key_len_ptr, + rhndl); + break; + + case KCF_OP_KEY_UNWRAP: + err = KCF_PROV_KEY_UNWRAP(pd, kops->ko_sid, + &kops->ko_mech, + kops->ko_key, kops->ko_wrapped_key, + kops->ko_wrapped_key_len_ptr, + kops->ko_key_template, kops->ko_key_attribute_count, + kops->ko_key_object_id_ptr, rhndl); + break; + + case KCF_OP_KEY_DERIVE: + err = KCF_PROV_KEY_DERIVE(pd, kops->ko_sid, + &kops->ko_mech, + kops->ko_key, kops->ko_key_template, + kops->ko_key_attribute_count, + kops->ko_key_object_id_ptr, rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_RANDOM: { + kcf_random_number_ops_params_t *rops = + ¶ms->rp_u.random_number_params; + + ASSERT(ctx == NULL); + + switch (optype) { + case KCF_OP_RANDOM_SEED: + err = KCF_PROV_SEED_RANDOM(pd, rops->rn_sid, + rops->rn_buf, rops->rn_buflen, rops->rn_entropy_est, + rops->rn_flags, rhndl); + break; + + case KCF_OP_RANDOM_GENERATE: + err = KCF_PROV_GENERATE_RANDOM(pd, rops->rn_sid, + rops->rn_buf, rops->rn_buflen, rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_SESSION: { + kcf_session_ops_params_t *sops = ¶ms->rp_u.session_params; + + ASSERT(ctx == NULL); + switch (optype) { + case KCF_OP_SESSION_OPEN: + /* + * so_pd may be a logical provider, in which case + * we need to check whether it has been removed. + */ + if (KCF_IS_PROV_REMOVED(sops->so_pd)) { + err = CRYPTO_DEVICE_ERROR; + break; + } + err = KCF_PROV_SESSION_OPEN(pd, sops->so_sid_ptr, + rhndl, sops->so_pd); + break; + + case KCF_OP_SESSION_CLOSE: + /* + * so_pd may be a logical provider, in which case + * we need to check whether it has been removed. + */ + if (KCF_IS_PROV_REMOVED(sops->so_pd)) { + err = CRYPTO_DEVICE_ERROR; + break; + } + err = KCF_PROV_SESSION_CLOSE(pd, sops->so_sid, + rhndl, sops->so_pd); + break; + + case KCF_OP_SESSION_LOGIN: + err = KCF_PROV_SESSION_LOGIN(pd, sops->so_sid, + sops->so_user_type, sops->so_pin, + sops->so_pin_len, rhndl); + break; + + case KCF_OP_SESSION_LOGOUT: + err = KCF_PROV_SESSION_LOGOUT(pd, sops->so_sid, rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_OBJECT: { + kcf_object_ops_params_t *jops = ¶ms->rp_u.object_params; + + ASSERT(ctx == NULL); + switch (optype) { + case KCF_OP_OBJECT_CREATE: + err = KCF_PROV_OBJECT_CREATE(pd, jops->oo_sid, + jops->oo_template, jops->oo_attribute_count, + jops->oo_object_id_ptr, rhndl); + break; + + case KCF_OP_OBJECT_COPY: + err = KCF_PROV_OBJECT_COPY(pd, jops->oo_sid, + jops->oo_object_id, + jops->oo_template, jops->oo_attribute_count, + jops->oo_object_id_ptr, rhndl); + break; + + case KCF_OP_OBJECT_DESTROY: + err = KCF_PROV_OBJECT_DESTROY(pd, jops->oo_sid, + jops->oo_object_id, rhndl); + break; + + case KCF_OP_OBJECT_GET_SIZE: + err = KCF_PROV_OBJECT_GET_SIZE(pd, jops->oo_sid, + jops->oo_object_id, jops->oo_object_size, rhndl); + break; + + case KCF_OP_OBJECT_GET_ATTRIBUTE_VALUE: + err = KCF_PROV_OBJECT_GET_ATTRIBUTE_VALUE(pd, + jops->oo_sid, jops->oo_object_id, + jops->oo_template, jops->oo_attribute_count, rhndl); + break; + + case KCF_OP_OBJECT_SET_ATTRIBUTE_VALUE: + err = KCF_PROV_OBJECT_SET_ATTRIBUTE_VALUE(pd, + jops->oo_sid, jops->oo_object_id, + jops->oo_template, jops->oo_attribute_count, rhndl); + break; + + case KCF_OP_OBJECT_FIND_INIT: + err = KCF_PROV_OBJECT_FIND_INIT(pd, jops->oo_sid, + jops->oo_template, jops->oo_attribute_count, + jops->oo_find_init_pp_ptr, rhndl); + break; + + case KCF_OP_OBJECT_FIND: + err = KCF_PROV_OBJECT_FIND(pd, jops->oo_find_pp, + jops->oo_object_id_ptr, jops->oo_max_object_count, + jops->oo_object_count_ptr, rhndl); + break; + + case KCF_OP_OBJECT_FIND_FINAL: + err = KCF_PROV_OBJECT_FIND_FINAL(pd, jops->oo_find_pp, + rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_PROVMGMT: { + kcf_provmgmt_ops_params_t *pops = ¶ms->rp_u.provmgmt_params; + + ASSERT(ctx == NULL); + switch (optype) { + case KCF_OP_MGMT_EXTINFO: + /* + * po_pd may be a logical provider, in which case + * we need to check whether it has been removed. + */ + if (KCF_IS_PROV_REMOVED(pops->po_pd)) { + err = CRYPTO_DEVICE_ERROR; + break; + } + err = KCF_PROV_EXT_INFO(pd, pops->po_ext_info, rhndl, + pops->po_pd); + break; + + case KCF_OP_MGMT_INITTOKEN: + err = KCF_PROV_INIT_TOKEN(pd, pops->po_pin, + pops->po_pin_len, pops->po_label, rhndl); + break; + + case KCF_OP_MGMT_INITPIN: + err = KCF_PROV_INIT_PIN(pd, pops->po_sid, pops->po_pin, + pops->po_pin_len, rhndl); + break; + + case KCF_OP_MGMT_SETPIN: + err = KCF_PROV_SET_PIN(pd, pops->po_sid, + pops->po_old_pin, pops->po_old_pin_len, + pops->po_pin, pops->po_pin_len, rhndl); + break; + + default: + break; + } + break; + } + + case KCF_OG_NOSTORE_KEY: { + kcf_key_ops_params_t *kops = ¶ms->rp_u.key_params; + + ASSERT(ctx == NULL); + KCF_SET_PROVIDER_MECHNUM(kops->ko_framework_mechtype, pd, + &kops->ko_mech); + + switch (optype) { + case KCF_OP_KEY_GENERATE: + err = KCF_PROV_NOSTORE_KEY_GENERATE(pd, kops->ko_sid, + &kops->ko_mech, kops->ko_key_template, + kops->ko_key_attribute_count, + kops->ko_out_template1, + kops->ko_out_attribute_count1, rhndl); + break; + + case KCF_OP_KEY_GENERATE_PAIR: + err = KCF_PROV_NOSTORE_KEY_GENERATE_PAIR(pd, + kops->ko_sid, &kops->ko_mech, + kops->ko_key_template, kops->ko_key_attribute_count, + kops->ko_private_key_template, + kops->ko_private_key_attribute_count, + kops->ko_out_template1, + kops->ko_out_attribute_count1, + kops->ko_out_template2, + kops->ko_out_attribute_count2, + rhndl); + break; + + case KCF_OP_KEY_DERIVE: + err = KCF_PROV_NOSTORE_KEY_DERIVE(pd, kops->ko_sid, + &kops->ko_mech, kops->ko_key, + kops->ko_key_template, + kops->ko_key_attribute_count, + kops->ko_out_template1, + kops->ko_out_attribute_count1, rhndl); + break; + + default: + break; + } + break; + } + default: + break; + } /* end of switch(params->rp_opgrp) */ + + KCF_PROV_INCRSTATS(pd, err); + return (err); +} + + +/* + * Emulate the call for a multipart dual ops with 2 single steps. + * This routine is always called in the context of a working thread + * running kcf_svc_do_run(). + * The single steps are submitted in a pure synchronous way (blocking). + * When this routine returns, kcf_svc_do_run() will call kcf_aop_done() + * so the originating consumer's callback gets invoked. kcf_aop_done() + * takes care of freeing the operation context. So, this routine does + * not free the operation context. + * + * The provider descriptor is assumed held by the callers. + */ +static int +kcf_emulate_dual(kcf_provider_desc_t *pd, crypto_ctx_t *ctx, + kcf_req_params_t *params) +{ + int err = CRYPTO_ARGUMENTS_BAD; + kcf_op_type_t optype; + size_t save_len; + off_t save_offset; + + optype = params->rp_optype; + + switch (params->rp_opgrp) { + case KCF_OG_ENCRYPT_MAC: { + kcf_encrypt_mac_ops_params_t *cmops = + ¶ms->rp_u.encrypt_mac_params; + kcf_context_t *encr_kcf_ctx; + crypto_ctx_t *mac_ctx; + kcf_req_params_t encr_params; + + encr_kcf_ctx = (kcf_context_t *)(ctx->cc_framework_private); + + switch (optype) { + case KCF_OP_INIT: { + encr_kcf_ctx->kc_secondctx = NULL; + + KCF_WRAP_ENCRYPT_OPS_PARAMS(&encr_params, KCF_OP_INIT, + pd->pd_sid, &cmops->em_encr_mech, + cmops->em_encr_key, NULL, NULL, + cmops->em_encr_templ); + + err = kcf_submit_request(pd, ctx, NULL, &encr_params, + B_FALSE); + + /* It can't be CRYPTO_QUEUED */ + if (err != CRYPTO_SUCCESS) { + break; + } + + err = crypto_mac_init(&cmops->em_mac_mech, + cmops->em_mac_key, cmops->em_mac_templ, + (crypto_context_t *)&mac_ctx, NULL); + + if (err == CRYPTO_SUCCESS) { + encr_kcf_ctx->kc_secondctx = (kcf_context_t *) + mac_ctx->cc_framework_private; + KCF_CONTEXT_REFHOLD((kcf_context_t *) + mac_ctx->cc_framework_private); + } + + break; + + } + case KCF_OP_UPDATE: { + crypto_dual_data_t *ct = cmops->em_ciphertext; + crypto_data_t *pt = cmops->em_plaintext; + kcf_context_t *mac_kcf_ctx = encr_kcf_ctx->kc_secondctx; + crypto_ctx_t *mac_ctx = &mac_kcf_ctx->kc_glbl_ctx; + + KCF_WRAP_ENCRYPT_OPS_PARAMS(&encr_params, KCF_OP_UPDATE, + pd->pd_sid, NULL, NULL, pt, (crypto_data_t *)ct, + NULL); + + err = kcf_submit_request(pd, ctx, NULL, &encr_params, + B_FALSE); + + /* It can't be CRYPTO_QUEUED */ + if (err != CRYPTO_SUCCESS) { + break; + } + + save_offset = ct->dd_offset1; + save_len = ct->dd_len1; + if (ct->dd_len2 == 0) { + /* + * The previous encrypt step was an + * accumulation only and didn't produce any + * partial output + */ + if (ct->dd_len1 == 0) + break; + + } else { + ct->dd_offset1 = ct->dd_offset2; + ct->dd_len1 = ct->dd_len2; + } + err = crypto_mac_update((crypto_context_t)mac_ctx, + (crypto_data_t *)ct, NULL); + + ct->dd_offset1 = save_offset; + ct->dd_len1 = save_len; + + break; + } + case KCF_OP_FINAL: { + crypto_dual_data_t *ct = cmops->em_ciphertext; + crypto_data_t *mac = cmops->em_mac; + kcf_context_t *mac_kcf_ctx = encr_kcf_ctx->kc_secondctx; + crypto_ctx_t *mac_ctx = &mac_kcf_ctx->kc_glbl_ctx; + crypto_context_t mac_context = mac_ctx; + + KCF_WRAP_ENCRYPT_OPS_PARAMS(&encr_params, KCF_OP_FINAL, + pd->pd_sid, NULL, NULL, NULL, (crypto_data_t *)ct, + NULL); + + err = kcf_submit_request(pd, ctx, NULL, &encr_params, + B_FALSE); + + /* It can't be CRYPTO_QUEUED */ + if (err != CRYPTO_SUCCESS) { + crypto_cancel_ctx(mac_context); + break; + } + + if (ct->dd_len2 > 0) { + save_offset = ct->dd_offset1; + save_len = ct->dd_len1; + ct->dd_offset1 = ct->dd_offset2; + ct->dd_len1 = ct->dd_len2; + + err = crypto_mac_update(mac_context, + (crypto_data_t *)ct, NULL); + + ct->dd_offset1 = save_offset; + ct->dd_len1 = save_len; + + if (err != CRYPTO_SUCCESS) { + crypto_cancel_ctx(mac_context); + return (err); + } + } + + /* and finally, collect the MAC */ + err = crypto_mac_final(mac_context, mac, NULL); + break; + } + + default: + break; + } + KCF_PROV_INCRSTATS(pd, err); + break; + } + case KCF_OG_MAC_DECRYPT: { + kcf_mac_decrypt_ops_params_t *mdops = + ¶ms->rp_u.mac_decrypt_params; + kcf_context_t *decr_kcf_ctx; + crypto_ctx_t *mac_ctx; + kcf_req_params_t decr_params; + + decr_kcf_ctx = (kcf_context_t *)(ctx->cc_framework_private); + + switch (optype) { + case KCF_OP_INIT: { + decr_kcf_ctx->kc_secondctx = NULL; + + err = crypto_mac_init(&mdops->md_mac_mech, + mdops->md_mac_key, mdops->md_mac_templ, + (crypto_context_t *)&mac_ctx, NULL); + + /* It can't be CRYPTO_QUEUED */ + if (err != CRYPTO_SUCCESS) { + break; + } + + KCF_WRAP_DECRYPT_OPS_PARAMS(&decr_params, KCF_OP_INIT, + pd->pd_sid, &mdops->md_decr_mech, + mdops->md_decr_key, NULL, NULL, + mdops->md_decr_templ); + + err = kcf_submit_request(pd, ctx, NULL, &decr_params, + B_FALSE); + + /* It can't be CRYPTO_QUEUED */ + if (err != CRYPTO_SUCCESS) { + crypto_cancel_ctx((crypto_context_t)mac_ctx); + break; + } + + decr_kcf_ctx->kc_secondctx = (kcf_context_t *) + mac_ctx->cc_framework_private; + KCF_CONTEXT_REFHOLD((kcf_context_t *) + mac_ctx->cc_framework_private); + + break; + default: + break; + + } + case KCF_OP_UPDATE: { + crypto_dual_data_t *ct = mdops->md_ciphertext; + crypto_data_t *pt = mdops->md_plaintext; + kcf_context_t *mac_kcf_ctx = decr_kcf_ctx->kc_secondctx; + crypto_ctx_t *mac_ctx = &mac_kcf_ctx->kc_glbl_ctx; + + err = crypto_mac_update((crypto_context_t)mac_ctx, + (crypto_data_t *)ct, NULL); + + if (err != CRYPTO_SUCCESS) + break; + + save_offset = ct->dd_offset1; + save_len = ct->dd_len1; + + /* zero ct->dd_len2 means decrypt everything */ + if (ct->dd_len2 > 0) { + ct->dd_offset1 = ct->dd_offset2; + ct->dd_len1 = ct->dd_len2; + } + + err = crypto_decrypt_update((crypto_context_t)ctx, + (crypto_data_t *)ct, pt, NULL); + + ct->dd_offset1 = save_offset; + ct->dd_len1 = save_len; + + break; + } + case KCF_OP_FINAL: { + crypto_data_t *pt = mdops->md_plaintext; + crypto_data_t *mac = mdops->md_mac; + kcf_context_t *mac_kcf_ctx = decr_kcf_ctx->kc_secondctx; + crypto_ctx_t *mac_ctx = &mac_kcf_ctx->kc_glbl_ctx; + + err = crypto_mac_final((crypto_context_t)mac_ctx, + mac, NULL); + + if (err != CRYPTO_SUCCESS) { + crypto_cancel_ctx(ctx); + break; + } + + /* Get the last chunk of plaintext */ + KCF_CONTEXT_REFHOLD(decr_kcf_ctx); + err = crypto_decrypt_final((crypto_context_t)ctx, pt, + NULL); + + break; + } + } + break; + } + default: + + break; + } /* end of switch(params->rp_opgrp) */ + + return (err); +} diff --git a/module/icp/core/kcf_mech_tabs.c b/module/icp/core/kcf_mech_tabs.c new file mode 100644 index 000000000000..3545f03eebf8 --- /dev/null +++ b/module/icp/core/kcf_mech_tabs.c @@ -0,0 +1,775 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include + +/* Cryptographic mechanisms tables and their access functions */ + +/* + * Internal numbers assigned to mechanisms are coded as follows: + * + * +----------------+----------------+ + * | mech. class | mech. index | + * <--- 32-bits --->+<--- 32-bits ---> + * + * the mech_class identifies the table the mechanism belongs to. + * mech_index is the index for that mechanism in the table. + * A mechanism belongs to exactly 1 table. + * The tables are: + * . digest_mechs_tab[] for the msg digest mechs. + * . cipher_mechs_tab[] for encrypt/decrypt and wrap/unwrap mechs. + * . mac_mechs_tab[] for MAC mechs. + * . sign_mechs_tab[] for sign & verify mechs. + * . keyops_mechs_tab[] for key/key pair generation, and key derivation. + * . misc_mechs_tab[] for mechs that don't belong to any of the above. + * + * There are no holes in the tables. + */ + +/* + * Locking conventions: + * -------------------- + * A global mutex, kcf_mech_tabs_lock, serializes writes to the + * mechanism table via kcf_create_mech_entry(). + * + * A mutex is associated with every entry of the tables. + * The mutex is acquired whenever the entry is accessed for + * 1) retrieving the mech_id (comparing the mech name) + * 2) finding a provider for an xxx_init() or atomic operation. + * 3) altering the mechs entry to add or remove a provider. + * + * In 2), after a provider is chosen, its prov_desc is held and the + * entry's mutex must be dropped. The provider's working function (SPI) is + * called outside the mech_entry's mutex. + * + * The number of providers for a particular mechanism is not expected to be + * long enough to justify the cost of using rwlocks, so the per-mechanism + * entry mutex won't be very *hot*. + * + * When both kcf_mech_tabs_lock and a mech_entry mutex need to be held, + * kcf_mech_tabs_lock must always be acquired first. + * + */ + + /* Mechanisms tables */ + + +/* RFE 4687834 Will deal with the extensibility of these tables later */ + +kcf_mech_entry_t kcf_digest_mechs_tab[KCF_MAXDIGEST]; +kcf_mech_entry_t kcf_cipher_mechs_tab[KCF_MAXCIPHER]; +kcf_mech_entry_t kcf_mac_mechs_tab[KCF_MAXMAC]; +kcf_mech_entry_t kcf_sign_mechs_tab[KCF_MAXSIGN]; +kcf_mech_entry_t kcf_keyops_mechs_tab[KCF_MAXKEYOPS]; +kcf_mech_entry_t kcf_misc_mechs_tab[KCF_MAXMISC]; + +kcf_mech_entry_tab_t kcf_mech_tabs_tab[KCF_LAST_OPSCLASS + 1] = { + {0, NULL}, /* No class zero */ + {KCF_MAXDIGEST, kcf_digest_mechs_tab}, + {KCF_MAXCIPHER, kcf_cipher_mechs_tab}, + {KCF_MAXMAC, kcf_mac_mechs_tab}, + {KCF_MAXSIGN, kcf_sign_mechs_tab}, + {KCF_MAXKEYOPS, kcf_keyops_mechs_tab}, + {KCF_MAXMISC, kcf_misc_mechs_tab} +}; + +/* + * Per-algorithm internal threasholds for the minimum input size of before + * offloading to hardware provider. + * Dispatching a crypto operation to a hardware provider entails paying the + * cost of an additional context switch. Measurments with Sun Accelerator 4000 + * shows that 512-byte jobs or smaller are better handled in software. + * There is room for refinement here. + * + */ +int kcf_md5_threshold = 512; +int kcf_sha1_threshold = 512; +int kcf_des_threshold = 512; +int kcf_des3_threshold = 512; +int kcf_aes_threshold = 512; +int kcf_bf_threshold = 512; +int kcf_rc4_threshold = 512; + +kmutex_t kcf_mech_tabs_lock; +static uint32_t kcf_gen_swprov = 0; + +int kcf_mech_hash_size = 256; +mod_hash_t *kcf_mech_hash; /* mech name to id hash */ + +static crypto_mech_type_t +kcf_mech_hash_find(char *mechname) +{ + mod_hash_val_t hv; + crypto_mech_type_t mt; + + mt = CRYPTO_MECH_INVALID; + if (mod_hash_find(kcf_mech_hash, (mod_hash_key_t)mechname, &hv) == 0) { + mt = *(crypto_mech_type_t *)hv; + ASSERT(mt != CRYPTO_MECH_INVALID); + } + + return (mt); +} + +void +kcf_destroy_mech_tabs(void) +{ + if (kcf_mech_hash) mod_hash_destroy_hash(kcf_mech_hash); +} + +/* + * kcf_init_mech_tabs() + * + * Called by the misc/kcf's _init() routine to initialize the tables + * of mech_entry's. + */ +void +kcf_init_mech_tabs(void) +{ + int i, max; + kcf_ops_class_t class; + kcf_mech_entry_t *me_tab; + + /* Initializes the mutex locks. */ + + mutex_init(&kcf_mech_tabs_lock, NULL, MUTEX_DEFAULT, NULL); + + /* Then the pre-defined mechanism entries */ + + /* Two digests */ + (void) strncpy(kcf_digest_mechs_tab[0].me_name, SUN_CKM_MD5, + CRYPTO_MAX_MECH_NAME); + kcf_digest_mechs_tab[0].me_threshold = kcf_md5_threshold; + + (void) strncpy(kcf_digest_mechs_tab[1].me_name, SUN_CKM_SHA1, + CRYPTO_MAX_MECH_NAME); + kcf_digest_mechs_tab[1].me_threshold = kcf_sha1_threshold; + + /* The symmetric ciphers in various modes */ + (void) strncpy(kcf_cipher_mechs_tab[0].me_name, SUN_CKM_DES_CBC, + CRYPTO_MAX_MECH_NAME); + kcf_cipher_mechs_tab[0].me_threshold = kcf_des_threshold; + + (void) strncpy(kcf_cipher_mechs_tab[1].me_name, SUN_CKM_DES3_CBC, + CRYPTO_MAX_MECH_NAME); + kcf_cipher_mechs_tab[1].me_threshold = kcf_des3_threshold; + + (void) strncpy(kcf_cipher_mechs_tab[2].me_name, SUN_CKM_DES_ECB, + CRYPTO_MAX_MECH_NAME); + kcf_cipher_mechs_tab[2].me_threshold = kcf_des_threshold; + + (void) strncpy(kcf_cipher_mechs_tab[3].me_name, SUN_CKM_DES3_ECB, + CRYPTO_MAX_MECH_NAME); + kcf_cipher_mechs_tab[3].me_threshold = kcf_des3_threshold; + + (void) strncpy(kcf_cipher_mechs_tab[4].me_name, SUN_CKM_BLOWFISH_CBC, + CRYPTO_MAX_MECH_NAME); + kcf_cipher_mechs_tab[4].me_threshold = kcf_bf_threshold; + + (void) strncpy(kcf_cipher_mechs_tab[5].me_name, SUN_CKM_BLOWFISH_ECB, + CRYPTO_MAX_MECH_NAME); + kcf_cipher_mechs_tab[5].me_threshold = kcf_bf_threshold; + + (void) strncpy(kcf_cipher_mechs_tab[6].me_name, SUN_CKM_AES_CBC, + CRYPTO_MAX_MECH_NAME); + kcf_cipher_mechs_tab[6].me_threshold = kcf_aes_threshold; + + (void) strncpy(kcf_cipher_mechs_tab[7].me_name, SUN_CKM_AES_ECB, + CRYPTO_MAX_MECH_NAME); + kcf_cipher_mechs_tab[7].me_threshold = kcf_aes_threshold; + + (void) strncpy(kcf_cipher_mechs_tab[8].me_name, SUN_CKM_RC4, + CRYPTO_MAX_MECH_NAME); + kcf_cipher_mechs_tab[8].me_threshold = kcf_rc4_threshold; + + + /* 4 HMACs */ + (void) strncpy(kcf_mac_mechs_tab[0].me_name, SUN_CKM_MD5_HMAC, + CRYPTO_MAX_MECH_NAME); + kcf_mac_mechs_tab[0].me_threshold = kcf_md5_threshold; + + (void) strncpy(kcf_mac_mechs_tab[1].me_name, SUN_CKM_MD5_HMAC_GENERAL, + CRYPTO_MAX_MECH_NAME); + kcf_mac_mechs_tab[1].me_threshold = kcf_md5_threshold; + + (void) strncpy(kcf_mac_mechs_tab[2].me_name, SUN_CKM_SHA1_HMAC, + CRYPTO_MAX_MECH_NAME); + kcf_mac_mechs_tab[2].me_threshold = kcf_sha1_threshold; + + (void) strncpy(kcf_mac_mechs_tab[3].me_name, SUN_CKM_SHA1_HMAC_GENERAL, + CRYPTO_MAX_MECH_NAME); + kcf_mac_mechs_tab[3].me_threshold = kcf_sha1_threshold; + + + /* 1 random number generation pseudo mechanism */ + (void) strncpy(kcf_misc_mechs_tab[0].me_name, SUN_RANDOM, + CRYPTO_MAX_MECH_NAME); + + kcf_mech_hash = mod_hash_create_strhash_nodtr("kcf mech2id hash", + kcf_mech_hash_size, mod_hash_null_valdtor); + + for (class = KCF_FIRST_OPSCLASS; class <= KCF_LAST_OPSCLASS; class++) { + max = kcf_mech_tabs_tab[class].met_size; + me_tab = kcf_mech_tabs_tab[class].met_tab; + for (i = 0; i < max; i++) { + mutex_init(&(me_tab[i].me_mutex), NULL, + MUTEX_DEFAULT, NULL); + if (me_tab[i].me_name[0] != 0) { + me_tab[i].me_mechid = KCF_MECHID(class, i); + (void) mod_hash_insert(kcf_mech_hash, + (mod_hash_key_t)me_tab[i].me_name, + (mod_hash_val_t)&(me_tab[i].me_mechid)); + } + } + } +} + +/* + * kcf_create_mech_entry() + * + * Arguments: + * . The class of mechanism. + * . the name of the new mechanism. + * + * Description: + * Creates a new mech_entry for a mechanism not yet known to the + * framework. + * This routine is called by kcf_add_mech_provider, which is + * in turn invoked for each mechanism supported by a provider. + * The'class' argument depends on the crypto_func_group_t bitmask + * in the registering provider's mech_info struct for this mechanism. + * When there is ambiguity in the mapping between the crypto_func_group_t + * and a class (dual ops, ...) the KCF_MISC_CLASS should be used. + * + * Context: + * User context only. + * + * Returns: + * KCF_INVALID_MECH_CLASS or KCF_INVALID_MECH_NAME if the class or + * the mechname is bogus. + * KCF_MECH_TAB_FULL when there is no room left in the mech. tabs. + * KCF_SUCCESS otherwise. + */ +static int +kcf_create_mech_entry(kcf_ops_class_t class, char *mechname) +{ + crypto_mech_type_t mt; + kcf_mech_entry_t *me_tab; + int i = 0, size; + + if ((class < KCF_FIRST_OPSCLASS) || (class > KCF_LAST_OPSCLASS)) + return (KCF_INVALID_MECH_CLASS); + + if ((mechname == NULL) || (mechname[0] == 0)) + return (KCF_INVALID_MECH_NAME); + /* + * First check if the mechanism is already in one of the tables. + * The mech_entry could be in another class. + */ + mutex_enter(&kcf_mech_tabs_lock); + mt = kcf_mech_hash_find(mechname); + if (mt != CRYPTO_MECH_INVALID) { + /* Nothing to do, regardless the suggested class. */ + mutex_exit(&kcf_mech_tabs_lock); + return (KCF_SUCCESS); + } + /* Now take the next unused mech entry in the class's tab */ + me_tab = kcf_mech_tabs_tab[class].met_tab; + size = kcf_mech_tabs_tab[class].met_size; + + while (i < size) { + mutex_enter(&(me_tab[i].me_mutex)); + if (me_tab[i].me_name[0] == 0) { + /* Found an empty spot */ + (void) strncpy(me_tab[i].me_name, mechname, + CRYPTO_MAX_MECH_NAME); + me_tab[i].me_name[CRYPTO_MAX_MECH_NAME-1] = '\0'; + me_tab[i].me_mechid = KCF_MECHID(class, i); + /* + * No a-priori information about the new mechanism, so + * the threshold is set to zero. + */ + me_tab[i].me_threshold = 0; + + mutex_exit(&(me_tab[i].me_mutex)); + /* Add the new mechanism to the hash table */ + (void) mod_hash_insert(kcf_mech_hash, + (mod_hash_key_t)me_tab[i].me_name, + (mod_hash_val_t)&(me_tab[i].me_mechid)); + break; + } + mutex_exit(&(me_tab[i].me_mutex)); + i++; + } + + mutex_exit(&kcf_mech_tabs_lock); + + if (i == size) { + return (KCF_MECH_TAB_FULL); + } + + return (KCF_SUCCESS); +} + +/* + * kcf_add_mech_provider() + * + * Arguments: + * . An index in to the provider mechanism array + * . A pointer to the provider descriptor + * . A storage for the kcf_prov_mech_desc_t the entry was added at. + * + * Description: + * Adds a new provider of a mechanism to the mechanism's mech_entry + * chain. + * + * Context: + * User context only. + * + * Returns + * KCF_SUCCESS on success + * KCF_MECH_TAB_FULL otherwise. + */ +int +kcf_add_mech_provider(short mech_indx, + kcf_provider_desc_t *prov_desc, kcf_prov_mech_desc_t **pmdpp) +{ + int error; + kcf_mech_entry_t *mech_entry = NULL; + crypto_mech_info_t *mech_info; + crypto_mech_type_t kcf_mech_type, mt; + kcf_prov_mech_desc_t *prov_mech, *prov_mech2; + crypto_func_group_t simple_fg_mask, dual_fg_mask; + crypto_mech_info_t *dmi; + crypto_mech_info_list_t *mil, *mil2; + kcf_mech_entry_t *me; + int i; + + ASSERT(prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + mech_info = &prov_desc->pd_mechanisms[mech_indx]; + + /* + * A mechanism belongs to exactly one mechanism table. + * Find the class corresponding to the function group flag of + * the mechanism. + */ + kcf_mech_type = kcf_mech_hash_find(mech_info->cm_mech_name); + if (kcf_mech_type == CRYPTO_MECH_INVALID) { + crypto_func_group_t fg = mech_info->cm_func_group_mask; + kcf_ops_class_t class; + + if (fg & CRYPTO_FG_DIGEST || fg & CRYPTO_FG_DIGEST_ATOMIC) + class = KCF_DIGEST_CLASS; + else if (fg & CRYPTO_FG_ENCRYPT || fg & CRYPTO_FG_DECRYPT || + fg & CRYPTO_FG_ENCRYPT_ATOMIC || + fg & CRYPTO_FG_DECRYPT_ATOMIC) + class = KCF_CIPHER_CLASS; + else if (fg & CRYPTO_FG_MAC || fg & CRYPTO_FG_MAC_ATOMIC) + class = KCF_MAC_CLASS; + else if (fg & CRYPTO_FG_SIGN || fg & CRYPTO_FG_VERIFY || + fg & CRYPTO_FG_SIGN_ATOMIC || + fg & CRYPTO_FG_VERIFY_ATOMIC || + fg & CRYPTO_FG_SIGN_RECOVER || + fg & CRYPTO_FG_VERIFY_RECOVER) + class = KCF_SIGN_CLASS; + else if (fg & CRYPTO_FG_GENERATE || + fg & CRYPTO_FG_GENERATE_KEY_PAIR || + fg & CRYPTO_FG_WRAP || fg & CRYPTO_FG_UNWRAP || + fg & CRYPTO_FG_DERIVE) + class = KCF_KEYOPS_CLASS; + else + class = KCF_MISC_CLASS; + + /* + * Attempt to create a new mech_entry for the specified + * mechanism. kcf_create_mech_entry() can handle the case + * where such an entry already exists. + */ + if ((error = kcf_create_mech_entry(class, + mech_info->cm_mech_name)) != KCF_SUCCESS) { + return (error); + } + /* get the KCF mech type that was assigned to the mechanism */ + kcf_mech_type = kcf_mech_hash_find(mech_info->cm_mech_name); + ASSERT(kcf_mech_type != CRYPTO_MECH_INVALID); + } + + error = kcf_get_mech_entry(kcf_mech_type, &mech_entry); + ASSERT(error == KCF_SUCCESS); + + /* allocate and initialize new kcf_prov_mech_desc */ + prov_mech = kmem_zalloc(sizeof (kcf_prov_mech_desc_t), KM_SLEEP); + bcopy(mech_info, &prov_mech->pm_mech_info, sizeof (crypto_mech_info_t)); + prov_mech->pm_prov_desc = prov_desc; + prov_desc->pd_mech_indx[KCF_MECH2CLASS(kcf_mech_type)] + [KCF_MECH2INDEX(kcf_mech_type)] = mech_indx; + + KCF_PROV_REFHOLD(prov_desc); + KCF_PROV_IREFHOLD(prov_desc); + + dual_fg_mask = mech_info->cm_func_group_mask & CRYPTO_FG_DUAL_MASK; + + if (dual_fg_mask == ((crypto_func_group_t)0)) + goto add_entry; + + simple_fg_mask = (mech_info->cm_func_group_mask & + CRYPTO_FG_SIMPLEOP_MASK) | CRYPTO_FG_RANDOM; + + for (i = 0; i < prov_desc->pd_mech_list_count; i++) { + dmi = &prov_desc->pd_mechanisms[i]; + + /* skip self */ + if (dmi->cm_mech_number == mech_info->cm_mech_number) + continue; + + /* skip if not a dual operation mechanism */ + if (!(dmi->cm_func_group_mask & dual_fg_mask) || + (dmi->cm_func_group_mask & simple_fg_mask)) + continue; + + mt = kcf_mech_hash_find(dmi->cm_mech_name); + if (mt == CRYPTO_MECH_INVALID) + continue; + + if (kcf_get_mech_entry(mt, &me) != KCF_SUCCESS) + continue; + + mil = kmem_zalloc(sizeof (*mil), KM_SLEEP); + mil2 = kmem_zalloc(sizeof (*mil2), KM_SLEEP); + + /* + * Ignore hard-coded entries in the mech table + * if the provider hasn't registered. + */ + mutex_enter(&me->me_mutex); + if (me->me_hw_prov_chain == NULL && me->me_sw_prov == NULL) { + mutex_exit(&me->me_mutex); + kmem_free(mil, sizeof (*mil)); + kmem_free(mil2, sizeof (*mil2)); + continue; + } + + /* + * Add other dual mechanisms that have registered + * with the framework to this mechanism's + * cross-reference list. + */ + mil->ml_mech_info = *dmi; /* struct assignment */ + mil->ml_kcf_mechid = mt; + + /* add to head of list */ + mil->ml_next = prov_mech->pm_mi_list; + prov_mech->pm_mi_list = mil; + + if (prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER) + prov_mech2 = me->me_hw_prov_chain; + else + prov_mech2 = me->me_sw_prov; + + if (prov_mech2 == NULL) { + kmem_free(mil2, sizeof (*mil2)); + mutex_exit(&me->me_mutex); + continue; + } + + /* + * Update all other cross-reference lists by + * adding this new mechanism. + */ + while (prov_mech2 != NULL) { + if (prov_mech2->pm_prov_desc == prov_desc) { + /* struct assignment */ + mil2->ml_mech_info = *mech_info; + mil2->ml_kcf_mechid = kcf_mech_type; + + /* add to head of list */ + mil2->ml_next = prov_mech2->pm_mi_list; + prov_mech2->pm_mi_list = mil2; + break; + } + prov_mech2 = prov_mech2->pm_next; + } + if (prov_mech2 == NULL) + kmem_free(mil2, sizeof (*mil2)); + + mutex_exit(&me->me_mutex); + } + +add_entry: + /* + * Add new kcf_prov_mech_desc at the front of HW providers + * chain. + */ + switch (prov_desc->pd_prov_type) { + + case CRYPTO_HW_PROVIDER: + mutex_enter(&mech_entry->me_mutex); + prov_mech->pm_me = mech_entry; + prov_mech->pm_next = mech_entry->me_hw_prov_chain; + mech_entry->me_hw_prov_chain = prov_mech; + mech_entry->me_num_hwprov++; + mutex_exit(&mech_entry->me_mutex); + break; + + case CRYPTO_SW_PROVIDER: + mutex_enter(&mech_entry->me_mutex); + if (mech_entry->me_sw_prov != NULL) { + /* + * There is already a SW provider for this mechanism. + * Since we allow only one SW provider per mechanism, + * report this condition. + */ + cmn_err(CE_WARN, "The cryptographic software provider " + "\"%s\" will not be used for %s. The provider " + "\"%s\" will be used for this mechanism " + "instead.", prov_desc->pd_description, + mech_info->cm_mech_name, + mech_entry->me_sw_prov->pm_prov_desc-> + pd_description); + KCF_PROV_REFRELE(prov_desc); + kmem_free(prov_mech, sizeof (kcf_prov_mech_desc_t)); + prov_mech = NULL; + } else { + /* + * Set the provider as the software provider for + * this mechanism. + */ + mech_entry->me_sw_prov = prov_mech; + + /* We'll wrap around after 4 billion registrations! */ + mech_entry->me_gen_swprov = kcf_gen_swprov++; + } + mutex_exit(&mech_entry->me_mutex); + break; + default: + break; + } + + *pmdpp = prov_mech; + + return (KCF_SUCCESS); +} + +/* + * kcf_remove_mech_provider() + * + * Arguments: + * . mech_name: the name of the mechanism. + * . prov_desc: The provider descriptor + * + * Description: + * Removes a provider from chain of provider descriptors. + * The provider is made unavailable to kernel consumers for the specified + * mechanism. + * + * Context: + * User context only. + */ +void +kcf_remove_mech_provider(char *mech_name, kcf_provider_desc_t *prov_desc) +{ + crypto_mech_type_t mech_type; + kcf_prov_mech_desc_t *prov_mech = NULL, *prov_chain; + kcf_prov_mech_desc_t **prev_entry_next; + kcf_mech_entry_t *mech_entry; + crypto_mech_info_list_t *mil, *mil2, *next, **prev_next; + + ASSERT(prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER); + + /* get the KCF mech type that was assigned to the mechanism */ + if ((mech_type = kcf_mech_hash_find(mech_name)) == + CRYPTO_MECH_INVALID) { + /* + * Provider was not allowed for this mech due to policy or + * configuration. + */ + return; + } + + /* get a ptr to the mech_entry that was created */ + if (kcf_get_mech_entry(mech_type, &mech_entry) != KCF_SUCCESS) { + /* + * Provider was not allowed for this mech due to policy or + * configuration. + */ + return; + } + + mutex_enter(&mech_entry->me_mutex); + + switch (prov_desc->pd_prov_type) { + + case CRYPTO_HW_PROVIDER: + /* find the provider in the mech_entry chain */ + prev_entry_next = &mech_entry->me_hw_prov_chain; + prov_mech = mech_entry->me_hw_prov_chain; + while (prov_mech != NULL && + prov_mech->pm_prov_desc != prov_desc) { + prev_entry_next = &prov_mech->pm_next; + prov_mech = prov_mech->pm_next; + } + + if (prov_mech == NULL) { + /* entry not found, simply return */ + mutex_exit(&mech_entry->me_mutex); + return; + } + + /* remove provider entry from mech_entry chain */ + *prev_entry_next = prov_mech->pm_next; + ASSERT(mech_entry->me_num_hwprov > 0); + mech_entry->me_num_hwprov--; + break; + + case CRYPTO_SW_PROVIDER: + if (mech_entry->me_sw_prov == NULL || + mech_entry->me_sw_prov->pm_prov_desc != prov_desc) { + /* not the software provider for this mechanism */ + mutex_exit(&mech_entry->me_mutex); + return; + } + prov_mech = mech_entry->me_sw_prov; + mech_entry->me_sw_prov = NULL; + break; + default: + break; + } + + mutex_exit(&mech_entry->me_mutex); + + /* Free the dual ops cross-reference lists */ + mil = prov_mech->pm_mi_list; + while (mil != NULL) { + next = mil->ml_next; + if (kcf_get_mech_entry(mil->ml_kcf_mechid, + &mech_entry) != KCF_SUCCESS) { + mil = next; + continue; + } + + mutex_enter(&mech_entry->me_mutex); + if (prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER) + prov_chain = mech_entry->me_hw_prov_chain; + else + prov_chain = mech_entry->me_sw_prov; + + while (prov_chain != NULL) { + if (prov_chain->pm_prov_desc == prov_desc) { + prev_next = &prov_chain->pm_mi_list; + mil2 = prov_chain->pm_mi_list; + while (mil2 != NULL && + mil2->ml_kcf_mechid != mech_type) { + prev_next = &mil2->ml_next; + mil2 = mil2->ml_next; + } + if (mil2 != NULL) { + *prev_next = mil2->ml_next; + kmem_free(mil2, sizeof (*mil2)); + } + break; + } + prov_chain = prov_chain->pm_next; + } + + mutex_exit(&mech_entry->me_mutex); + kmem_free(mil, sizeof (crypto_mech_info_list_t)); + mil = next; + } + + /* free entry */ + KCF_PROV_REFRELE(prov_mech->pm_prov_desc); + KCF_PROV_IREFRELE(prov_mech->pm_prov_desc); + kmem_free(prov_mech, sizeof (kcf_prov_mech_desc_t)); +} + +/* + * kcf_get_mech_entry() + * + * Arguments: + * . The framework mechanism type + * . Storage for the mechanism entry + * + * Description: + * Retrieves the mechanism entry for the mech. + * + * Context: + * User and interrupt contexts. + * + * Returns: + * KCF_MECHANISM_XXX appropriate error code. + * KCF_SUCCESS otherwise. + */ +int +kcf_get_mech_entry(crypto_mech_type_t mech_type, kcf_mech_entry_t **mep) +{ + kcf_ops_class_t class; + int index; + kcf_mech_entry_tab_t *me_tab; + + ASSERT(mep != NULL); + + class = KCF_MECH2CLASS(mech_type); + + if ((class < KCF_FIRST_OPSCLASS) || (class > KCF_LAST_OPSCLASS)) { + /* the caller won't need to know it's an invalid class */ + return (KCF_INVALID_MECH_NUMBER); + } + + me_tab = &kcf_mech_tabs_tab[class]; + index = KCF_MECH2INDEX(mech_type); + + if ((index < 0) || (index >= me_tab->met_size)) { + return (KCF_INVALID_MECH_NUMBER); + } + + *mep = &((me_tab->met_tab)[index]); + + return (KCF_SUCCESS); +} + +/* CURRENTLY UNSUPPORTED: attempting to load the module if it isn't found */ +/* + * Lookup the hash table for an entry that matches the mechname. + * If there are no hardware or software providers for the mechanism, + * but there is an unloaded software provider, this routine will attempt + * to load it. + * + * If the MOD_NOAUTOUNLOAD flag is not set, a software provider is + * in constant danger of being unloaded. For consumers that call + * crypto_mech2id() only once, the provider will not be reloaded + * if it becomes unloaded. If a provider gets loaded elsewhere + * without the MOD_NOAUTOUNLOAD flag being set, we set it now. + */ +crypto_mech_type_t +crypto_mech2id_common(char *mechname, boolean_t load_module) +{ + crypto_mech_type_t mt = kcf_mech_hash_find(mechname); + return (mt); +} diff --git a/module/icp/core/kcf_prov_lib.c b/module/icp/core/kcf_prov_lib.c new file mode 100644 index 000000000000..dd4cd086d21f --- /dev/null +++ b/module/icp/core/kcf_prov_lib.c @@ -0,0 +1,229 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include + +/* + * Utility routine to copy a buffer to a crypto_data structure. + */ + +/* + * Utility routine to apply the command, 'cmd', to the + * data in the uio structure. + */ +int +crypto_uio_data(crypto_data_t *data, uchar_t *buf, int len, cmd_type_t cmd, + void *digest_ctx, void (*update)(void)) +{ + uio_t *uiop = data->cd_uio; + off_t offset = data->cd_offset; + size_t length = len; + uint_t vec_idx; + size_t cur_len; + uchar_t *datap; + + ASSERT(data->cd_format == CRYPTO_DATA_UIO); + if (uiop->uio_segflg != UIO_SYSSPACE) { + return (CRYPTO_ARGUMENTS_BAD); + } + + /* + * Jump to the first iovec containing data to be + * processed. + */ + for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && + offset >= uiop->uio_iov[vec_idx].iov_len; + offset -= uiop->uio_iov[vec_idx++].iov_len) + ; + + if (vec_idx == uiop->uio_iovcnt) { + /* + * The caller specified an offset that is larger than + * the total size of the buffers it provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + + while (vec_idx < uiop->uio_iovcnt && length > 0) { + cur_len = MIN(uiop->uio_iov[vec_idx].iov_len - + offset, length); + + datap = (uchar_t *)(uiop->uio_iov[vec_idx].iov_base + + offset); + switch (cmd) { + case COPY_FROM_DATA: + bcopy(datap, buf, cur_len); + buf += cur_len; + break; + case COPY_TO_DATA: + bcopy(buf, datap, cur_len); + buf += cur_len; + break; + case COMPARE_TO_DATA: + if (bcmp(datap, buf, cur_len)) + return (CRYPTO_SIGNATURE_INVALID); + buf += cur_len; + break; + case MD5_DIGEST_DATA: + case SHA1_DIGEST_DATA: + case SHA2_DIGEST_DATA: + case GHASH_DATA: + return (CRYPTO_ARGUMENTS_BAD); + } + + length -= cur_len; + vec_idx++; + offset = 0; + } + + if (vec_idx == uiop->uio_iovcnt && length > 0) { + /* + * The end of the specified iovec's was reached but + * the length requested could not be processed. + */ + switch (cmd) { + case COPY_TO_DATA: + data->cd_length = len; + return (CRYPTO_BUFFER_TOO_SMALL); + default: + return (CRYPTO_DATA_LEN_RANGE); + } + } + + return (CRYPTO_SUCCESS); +} + +int +crypto_put_output_data(uchar_t *buf, crypto_data_t *output, int len) +{ + switch (output->cd_format) { + case CRYPTO_DATA_RAW: + if (output->cd_raw.iov_len < len) { + output->cd_length = len; + return (CRYPTO_BUFFER_TOO_SMALL); + } + bcopy(buf, (uchar_t *)(output->cd_raw.iov_base + + output->cd_offset), len); + break; + + case CRYPTO_DATA_UIO: + return (crypto_uio_data(output, buf, len, + COPY_TO_DATA, NULL, NULL)); + default: + return (CRYPTO_ARGUMENTS_BAD); + } + + return (CRYPTO_SUCCESS); +} + +int +crypto_update_iov(void *ctx, crypto_data_t *input, crypto_data_t *output, + int (*cipher)(void *, caddr_t, size_t, crypto_data_t *), + void (*copy_block)(uint8_t *, uint64_t *)) +{ + common_ctx_t *common_ctx = ctx; + int rv; + + if (input->cd_miscdata != NULL) { + copy_block((uint8_t *)input->cd_miscdata, + &common_ctx->cc_iv[0]); + } + + if (input->cd_raw.iov_len < input->cd_length) + return (CRYPTO_ARGUMENTS_BAD); + + rv = (cipher)(ctx, input->cd_raw.iov_base + input->cd_offset, + input->cd_length, (input == output) ? NULL : output); + + return (rv); +} + +int +crypto_update_uio(void *ctx, crypto_data_t *input, crypto_data_t *output, + int (*cipher)(void *, caddr_t, size_t, crypto_data_t *), + void (*copy_block)(uint8_t *, uint64_t *)) +{ + common_ctx_t *common_ctx = ctx; + uio_t *uiop = input->cd_uio; + off_t offset = input->cd_offset; + size_t length = input->cd_length; + uint_t vec_idx; + size_t cur_len; + + if (input->cd_miscdata != NULL) { + copy_block((uint8_t *)input->cd_miscdata, + &common_ctx->cc_iv[0]); + } + + if (input->cd_uio->uio_segflg != UIO_SYSSPACE) { + return (CRYPTO_ARGUMENTS_BAD); + } + + /* + * Jump to the first iovec containing data to be + * processed. + */ + for (vec_idx = 0; vec_idx < uiop->uio_iovcnt && + offset >= uiop->uio_iov[vec_idx].iov_len; + offset -= uiop->uio_iov[vec_idx++].iov_len) + ; + if (vec_idx == uiop->uio_iovcnt) { + /* + * The caller specified an offset that is larger than the + * total size of the buffers it provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + + /* + * Now process the iovecs. + */ + while (vec_idx < uiop->uio_iovcnt && length > 0) { + cur_len = MIN(uiop->uio_iov[vec_idx].iov_len - + offset, length); + + (cipher)(ctx, uiop->uio_iov[vec_idx].iov_base + offset, + cur_len, (input == output) ? NULL : output); + + length -= cur_len; + vec_idx++; + offset = 0; + } + + if (vec_idx == uiop->uio_iovcnt && length > 0) { + /* + * The end of the specified iovec's was reached but + * the length requested could not be processed, i.e. + * The caller requested to digest more data than it provided. + */ + + return (CRYPTO_DATA_LEN_RANGE); + } + + return (CRYPTO_SUCCESS); +} diff --git a/module/icp/core/kcf_prov_tabs.c b/module/icp/core/kcf_prov_tabs.c new file mode 100644 index 000000000000..dca0fc1038c7 --- /dev/null +++ b/module/icp/core/kcf_prov_tabs.c @@ -0,0 +1,638 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * This file is part of the core Kernel Cryptographic Framework. + * It implements the management of tables of Providers. Entries to + * added and removed when cryptographic providers register with + * and unregister from the framework, respectively. The KCF scheduler + * and ioctl pseudo driver call this function to obtain the list + * of available providers. + * + * The provider table is indexed by crypto_provider_id_t. Each + * element of the table contains a pointer to a provider descriptor, + * or NULL if the entry is free. + * + * This file also implements helper functions to allocate and free + * provider descriptors. + */ + +#include +#include +#include +#include +#include + +#define KCF_MAX_PROVIDERS 512 /* max number of providers */ + +/* + * Prov_tab is an array of providers which is updated when + * a crypto provider registers with kcf. The provider calls the + * SPI routine, crypto_register_provider(), which in turn calls + * kcf_prov_tab_add_provider(). + * + * A provider unregisters by calling crypto_unregister_provider() + * which triggers the removal of the prov_tab entry. + * It also calls kcf_remove_mech_provider(). + * + * prov_tab entries are not updated from kcf.conf or by cryptoadm(1M). + */ +static kcf_provider_desc_t **prov_tab = NULL; +static kmutex_t prov_tab_mutex; /* ensure exclusive access to the table */ +static uint_t prov_tab_num = 0; /* number of providers in table */ +static uint_t prov_tab_max = KCF_MAX_PROVIDERS; + +void +kcf_prov_tab_destroy(void) +{ + if (prov_tab) kmem_free(prov_tab, prov_tab_max * + sizeof (kcf_provider_desc_t *)); +} + +/* + * Initialize a mutex and the KCF providers table, prov_tab. + * The providers table is dynamically allocated with prov_tab_max entries. + * Called from kcf module _init(). + */ +void +kcf_prov_tab_init(void) +{ + mutex_init(&prov_tab_mutex, NULL, MUTEX_DEFAULT, NULL); + + prov_tab = kmem_zalloc(prov_tab_max * sizeof (kcf_provider_desc_t *), + KM_SLEEP); +} + +/* + * Add a provider to the provider table. If no free entry can be found + * for the new provider, returns CRYPTO_HOST_MEMORY. Otherwise, add + * the provider to the table, initialize the pd_prov_id field + * of the specified provider descriptor to the index in that table, + * and return CRYPTO_SUCCESS. Note that a REFHOLD is done on the + * provider when pointed to by a table entry. + */ +int +kcf_prov_tab_add_provider(kcf_provider_desc_t *prov_desc) +{ + uint_t i; + + ASSERT(prov_tab != NULL); + + mutex_enter(&prov_tab_mutex); + + /* find free slot in providers table */ + for (i = 1; i < KCF_MAX_PROVIDERS && prov_tab[i] != NULL; i++) + ; + if (i == KCF_MAX_PROVIDERS) { + /* ran out of providers entries */ + mutex_exit(&prov_tab_mutex); + cmn_err(CE_WARN, "out of providers entries"); + return (CRYPTO_HOST_MEMORY); + } + + /* initialize entry */ + prov_tab[i] = prov_desc; + KCF_PROV_REFHOLD(prov_desc); + KCF_PROV_IREFHOLD(prov_desc); + prov_tab_num++; + + mutex_exit(&prov_tab_mutex); + + /* update provider descriptor */ + prov_desc->pd_prov_id = i; + + /* + * The KCF-private provider handle is defined as the internal + * provider id. + */ + prov_desc->pd_kcf_prov_handle = + (crypto_kcf_provider_handle_t)prov_desc->pd_prov_id; + + return (CRYPTO_SUCCESS); +} + +/* + * Remove the provider specified by its id. A REFRELE is done on the + * corresponding provider descriptor before this function returns. + * Returns CRYPTO_UNKNOWN_PROVIDER if the provider id is not valid. + */ +int +kcf_prov_tab_rem_provider(crypto_provider_id_t prov_id) +{ + kcf_provider_desc_t *prov_desc; + + ASSERT(prov_tab != NULL); + ASSERT(prov_tab_num >= 0); + + /* + * Validate provider id, since it can be specified by a 3rd-party + * provider. + */ + + mutex_enter(&prov_tab_mutex); + if (prov_id >= KCF_MAX_PROVIDERS || + ((prov_desc = prov_tab[prov_id]) == NULL)) { + mutex_exit(&prov_tab_mutex); + return (CRYPTO_INVALID_PROVIDER_ID); + } + mutex_exit(&prov_tab_mutex); + + /* + * The provider id must remain valid until the associated provider + * descriptor is freed. For this reason, we simply release our + * reference to the descriptor here. When the reference count + * reaches zero, kcf_free_provider_desc() will be invoked and + * the associated entry in the providers table will be released + * at that time. + */ + + KCF_PROV_REFRELE(prov_desc); + KCF_PROV_IREFRELE(prov_desc); + + return (CRYPTO_SUCCESS); +} + +/* + * Returns the provider descriptor corresponding to the specified + * provider id. A REFHOLD is done on the descriptor before it is + * returned to the caller. It is the responsibility of the caller + * to do a REFRELE once it is done with the provider descriptor. + */ +kcf_provider_desc_t * +kcf_prov_tab_lookup(crypto_provider_id_t prov_id) +{ + kcf_provider_desc_t *prov_desc; + + mutex_enter(&prov_tab_mutex); + + prov_desc = prov_tab[prov_id]; + + if (prov_desc == NULL) { + mutex_exit(&prov_tab_mutex); + return (NULL); + } + + KCF_PROV_REFHOLD(prov_desc); + + mutex_exit(&prov_tab_mutex); + + return (prov_desc); +} + +static void +allocate_ops_v1(crypto_ops_t *src, crypto_ops_t *dst, uint_t *mech_list_count) +{ + if (src->co_control_ops != NULL) + dst->co_control_ops = kmem_alloc(sizeof (crypto_control_ops_t), + KM_SLEEP); + + if (src->co_digest_ops != NULL) + dst->co_digest_ops = kmem_alloc(sizeof (crypto_digest_ops_t), + KM_SLEEP); + + if (src->co_cipher_ops != NULL) + dst->co_cipher_ops = kmem_alloc(sizeof (crypto_cipher_ops_t), + KM_SLEEP); + + if (src->co_mac_ops != NULL) + dst->co_mac_ops = kmem_alloc(sizeof (crypto_mac_ops_t), + KM_SLEEP); + + if (src->co_sign_ops != NULL) + dst->co_sign_ops = kmem_alloc(sizeof (crypto_sign_ops_t), + KM_SLEEP); + + if (src->co_verify_ops != NULL) + dst->co_verify_ops = kmem_alloc(sizeof (crypto_verify_ops_t), + KM_SLEEP); + + if (src->co_dual_ops != NULL) + dst->co_dual_ops = kmem_alloc(sizeof (crypto_dual_ops_t), + KM_SLEEP); + + if (src->co_dual_cipher_mac_ops != NULL) + dst->co_dual_cipher_mac_ops = kmem_alloc( + sizeof (crypto_dual_cipher_mac_ops_t), KM_SLEEP); + + if (src->co_random_ops != NULL) { + dst->co_random_ops = kmem_alloc( + sizeof (crypto_random_number_ops_t), KM_SLEEP); + + /* + * Allocate storage to store the array of supported mechanisms + * specified by provider. We allocate extra mechanism storage + * if the provider has random_ops since we keep an internal + * mechanism, SUN_RANDOM, in this case. + */ + (*mech_list_count)++; + } + + if (src->co_session_ops != NULL) + dst->co_session_ops = kmem_alloc(sizeof (crypto_session_ops_t), + KM_SLEEP); + + if (src->co_object_ops != NULL) + dst->co_object_ops = kmem_alloc(sizeof (crypto_object_ops_t), + KM_SLEEP); + + if (src->co_key_ops != NULL) + dst->co_key_ops = kmem_alloc(sizeof (crypto_key_ops_t), + KM_SLEEP); + + if (src->co_provider_ops != NULL) + dst->co_provider_ops = kmem_alloc( + sizeof (crypto_provider_management_ops_t), KM_SLEEP); + + if (src->co_ctx_ops != NULL) + dst->co_ctx_ops = kmem_alloc(sizeof (crypto_ctx_ops_t), + KM_SLEEP); +} + +static void +allocate_ops_v2(crypto_ops_t *src, crypto_ops_t *dst) +{ + if (src->co_mech_ops != NULL) + dst->co_mech_ops = kmem_alloc(sizeof (crypto_mech_ops_t), + KM_SLEEP); +} + +static void +allocate_ops_v3(crypto_ops_t *src, crypto_ops_t *dst) +{ + if (src->co_nostore_key_ops != NULL) + dst->co_nostore_key_ops = + kmem_alloc(sizeof (crypto_nostore_key_ops_t), KM_SLEEP); +} + +/* + * Allocate a provider descriptor. mech_list_count specifies the + * number of mechanisms supported by the providers, and is used + * to allocate storage for the mechanism table. + * This function may sleep while allocating memory, which is OK + * since it is invoked from user context during provider registration. + */ +kcf_provider_desc_t * +kcf_alloc_provider_desc(crypto_provider_info_t *info) +{ + int i, j; + kcf_provider_desc_t *desc; + uint_t mech_list_count = info->pi_mech_list_count; + crypto_ops_t *src_ops = info->pi_ops_vector; + + desc = kmem_zalloc(sizeof (kcf_provider_desc_t), KM_SLEEP); + + /* + * pd_description serves two purposes + * - Appears as a blank padded PKCS#11 style string, that will be + * returned to applications in CK_SLOT_INFO.slotDescription. + * This means that we should not have a null character in the + * first CRYPTO_PROVIDER_DESCR_MAX_LEN bytes. + * - Appears as a null-terminated string that can be used by + * other kcf routines. + * + * So, we allocate enough room for one extra null terminator + * which keeps every one happy. + */ + desc->pd_description = kmem_alloc(CRYPTO_PROVIDER_DESCR_MAX_LEN + 1, + KM_SLEEP); + (void) memset(desc->pd_description, ' ', + CRYPTO_PROVIDER_DESCR_MAX_LEN); + desc->pd_description[CRYPTO_PROVIDER_DESCR_MAX_LEN] = '\0'; + + /* + * Since the framework does not require the ops vector specified + * by the providers during registration to be persistent, + * KCF needs to allocate storage where copies of the ops + * vectors are copied. + */ + desc->pd_ops_vector = kmem_zalloc(sizeof (crypto_ops_t), KM_SLEEP); + + if (info->pi_provider_type != CRYPTO_LOGICAL_PROVIDER) { + allocate_ops_v1(src_ops, desc->pd_ops_vector, &mech_list_count); + if (info->pi_interface_version >= CRYPTO_SPI_VERSION_2) + allocate_ops_v2(src_ops, desc->pd_ops_vector); + if (info->pi_interface_version == CRYPTO_SPI_VERSION_3) + allocate_ops_v3(src_ops, desc->pd_ops_vector); + } + + desc->pd_mech_list_count = mech_list_count; + desc->pd_mechanisms = kmem_zalloc(sizeof (crypto_mech_info_t) * + mech_list_count, KM_SLEEP); + for (i = 0; i < KCF_OPS_CLASSSIZE; i++) + for (j = 0; j < KCF_MAXMECHTAB; j++) + desc->pd_mech_indx[i][j] = KCF_INVALID_INDX; + + desc->pd_prov_id = KCF_PROVID_INVALID; + desc->pd_state = KCF_PROV_ALLOCATED; + + mutex_init(&desc->pd_lock, NULL, MUTEX_DEFAULT, NULL); + cv_init(&desc->pd_resume_cv, NULL, CV_DEFAULT, NULL); + cv_init(&desc->pd_remove_cv, NULL, CV_DEFAULT, NULL); + + return (desc); +} + +/* + * Called by KCF_PROV_REFRELE when a provider's reference count drops + * to zero. We free the descriptor when the last reference is released. + * However, for software providers, we do not free it when there is an + * unregister thread waiting. We signal that thread in this case and + * that thread is responsible for freeing the descriptor. + */ +void +kcf_provider_zero_refcnt(kcf_provider_desc_t *desc) +{ + mutex_enter(&desc->pd_lock); + switch (desc->pd_prov_type) { + case CRYPTO_SW_PROVIDER: + if (desc->pd_state == KCF_PROV_REMOVED || + desc->pd_state == KCF_PROV_DISABLED) { + desc->pd_state = KCF_PROV_FREED; + cv_broadcast(&desc->pd_remove_cv); + mutex_exit(&desc->pd_lock); + break; + } + /* FALLTHRU */ + + case CRYPTO_HW_PROVIDER: + case CRYPTO_LOGICAL_PROVIDER: + mutex_exit(&desc->pd_lock); + kcf_free_provider_desc(desc); + } +} + +/* + * Free a provider descriptor. + */ +void +kcf_free_provider_desc(kcf_provider_desc_t *desc) +{ + if (desc == NULL) + return; + + mutex_enter(&prov_tab_mutex); + if (desc->pd_prov_id != KCF_PROVID_INVALID) { + /* release the associated providers table entry */ + ASSERT(prov_tab[desc->pd_prov_id] != NULL); + prov_tab[desc->pd_prov_id] = NULL; + prov_tab_num--; + } + mutex_exit(&prov_tab_mutex); + + /* free the kernel memory associated with the provider descriptor */ + + if (desc->pd_description != NULL) + kmem_free(desc->pd_description, + CRYPTO_PROVIDER_DESCR_MAX_LEN + 1); + + if (desc->pd_ops_vector != NULL) { + + if (desc->pd_ops_vector->co_control_ops != NULL) + kmem_free(desc->pd_ops_vector->co_control_ops, + sizeof (crypto_control_ops_t)); + + if (desc->pd_ops_vector->co_digest_ops != NULL) + kmem_free(desc->pd_ops_vector->co_digest_ops, + sizeof (crypto_digest_ops_t)); + + if (desc->pd_ops_vector->co_cipher_ops != NULL) + kmem_free(desc->pd_ops_vector->co_cipher_ops, + sizeof (crypto_cipher_ops_t)); + + if (desc->pd_ops_vector->co_mac_ops != NULL) + kmem_free(desc->pd_ops_vector->co_mac_ops, + sizeof (crypto_mac_ops_t)); + + if (desc->pd_ops_vector->co_sign_ops != NULL) + kmem_free(desc->pd_ops_vector->co_sign_ops, + sizeof (crypto_sign_ops_t)); + + if (desc->pd_ops_vector->co_verify_ops != NULL) + kmem_free(desc->pd_ops_vector->co_verify_ops, + sizeof (crypto_verify_ops_t)); + + if (desc->pd_ops_vector->co_dual_ops != NULL) + kmem_free(desc->pd_ops_vector->co_dual_ops, + sizeof (crypto_dual_ops_t)); + + if (desc->pd_ops_vector->co_dual_cipher_mac_ops != NULL) + kmem_free(desc->pd_ops_vector->co_dual_cipher_mac_ops, + sizeof (crypto_dual_cipher_mac_ops_t)); + + if (desc->pd_ops_vector->co_random_ops != NULL) + kmem_free(desc->pd_ops_vector->co_random_ops, + sizeof (crypto_random_number_ops_t)); + + if (desc->pd_ops_vector->co_session_ops != NULL) + kmem_free(desc->pd_ops_vector->co_session_ops, + sizeof (crypto_session_ops_t)); + + if (desc->pd_ops_vector->co_object_ops != NULL) + kmem_free(desc->pd_ops_vector->co_object_ops, + sizeof (crypto_object_ops_t)); + + if (desc->pd_ops_vector->co_key_ops != NULL) + kmem_free(desc->pd_ops_vector->co_key_ops, + sizeof (crypto_key_ops_t)); + + if (desc->pd_ops_vector->co_provider_ops != NULL) + kmem_free(desc->pd_ops_vector->co_provider_ops, + sizeof (crypto_provider_management_ops_t)); + + if (desc->pd_ops_vector->co_ctx_ops != NULL) + kmem_free(desc->pd_ops_vector->co_ctx_ops, + sizeof (crypto_ctx_ops_t)); + + if (desc->pd_ops_vector->co_mech_ops != NULL) + kmem_free(desc->pd_ops_vector->co_mech_ops, + sizeof (crypto_mech_ops_t)); + + if (desc->pd_ops_vector->co_nostore_key_ops != NULL) + kmem_free(desc->pd_ops_vector->co_nostore_key_ops, + sizeof (crypto_nostore_key_ops_t)); + + kmem_free(desc->pd_ops_vector, sizeof (crypto_ops_t)); + } + + if (desc->pd_mechanisms != NULL) + /* free the memory associated with the mechanism info's */ + kmem_free(desc->pd_mechanisms, sizeof (crypto_mech_info_t) * + desc->pd_mech_list_count); + + if (desc->pd_sched_info.ks_taskq != NULL) + taskq_destroy(desc->pd_sched_info.ks_taskq); + + kmem_free(desc, sizeof (kcf_provider_desc_t)); +} + +/* + * Returns an array of hardware and logical provider descriptors, + * a.k.a the PKCS#11 slot list. A REFHOLD is done on each descriptor + * before the array is returned. The entire table can be freed by + * calling kcf_free_provider_tab(). + */ +int +kcf_get_slot_list(uint_t *count, kcf_provider_desc_t ***array, + boolean_t unverified) +{ + kcf_provider_desc_t *prov_desc; + kcf_provider_desc_t **p = NULL; + char *last; + uint_t cnt = 0; + uint_t i, j; + int rval = CRYPTO_SUCCESS; + size_t n, final_size; + + /* count the providers */ + mutex_enter(&prov_tab_mutex); + for (i = 0; i < KCF_MAX_PROVIDERS; i++) { + if ((prov_desc = prov_tab[i]) != NULL && + ((prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER && + (prov_desc->pd_flags & CRYPTO_HIDE_PROVIDER) == 0) || + prov_desc->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)) { + if (KCF_IS_PROV_USABLE(prov_desc) || + (unverified && KCF_IS_PROV_UNVERIFIED(prov_desc))) { + cnt++; + } + } + } + mutex_exit(&prov_tab_mutex); + + if (cnt == 0) + goto out; + + n = cnt * sizeof (kcf_provider_desc_t *); +again: + p = kmem_zalloc(n, KM_SLEEP); + + /* pointer to last entry in the array */ + last = (char *)&p[cnt-1]; + + mutex_enter(&prov_tab_mutex); + /* fill the slot list */ + for (i = 0, j = 0; i < KCF_MAX_PROVIDERS; i++) { + if ((prov_desc = prov_tab[i]) != NULL && + ((prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER && + (prov_desc->pd_flags & CRYPTO_HIDE_PROVIDER) == 0) || + prov_desc->pd_prov_type == CRYPTO_LOGICAL_PROVIDER)) { + if (KCF_IS_PROV_USABLE(prov_desc) || + (unverified && KCF_IS_PROV_UNVERIFIED(prov_desc))) { + if ((char *)&p[j] > last) { + mutex_exit(&prov_tab_mutex); + kcf_free_provider_tab(cnt, p); + n = n << 1; + cnt = cnt << 1; + goto again; + } + p[j++] = prov_desc; + KCF_PROV_REFHOLD(prov_desc); + } + } + } + mutex_exit(&prov_tab_mutex); + + final_size = j * sizeof (kcf_provider_desc_t *); + cnt = j; + ASSERT(final_size <= n); + + /* check if buffer we allocated is too large */ + if (final_size < n) { + char *final_buffer = NULL; + + if (final_size > 0) { + final_buffer = kmem_alloc(final_size, KM_SLEEP); + bcopy(p, final_buffer, final_size); + } + kmem_free(p, n); + p = (kcf_provider_desc_t **)final_buffer; + } +out: + *count = cnt; + *array = p; + return (rval); +} + +/* + * Free an array of hardware provider descriptors. A REFRELE + * is done on each descriptor before the table is freed. + */ +void +kcf_free_provider_tab(uint_t count, kcf_provider_desc_t **array) +{ + kcf_provider_desc_t *prov_desc; + int i; + + for (i = 0; i < count; i++) { + if ((prov_desc = array[i]) != NULL) { + KCF_PROV_REFRELE(prov_desc); + } + } + kmem_free(array, count * sizeof (kcf_provider_desc_t *)); +} + +/* + * Returns in the location pointed to by pd a pointer to the descriptor + * for the software provider for the specified mechanism. + * The provider descriptor is returned held and it is the caller's + * responsibility to release it when done. The mechanism entry + * is returned if the optional argument mep is non NULL. + * + * Returns one of the CRYPTO_ * error codes on failure, and + * CRYPTO_SUCCESS on success. + */ +int +kcf_get_sw_prov(crypto_mech_type_t mech_type, kcf_provider_desc_t **pd, + kcf_mech_entry_t **mep, boolean_t log_warn) +{ + kcf_mech_entry_t *me; + + /* get the mechanism entry for this mechanism */ + if (kcf_get_mech_entry(mech_type, &me) != KCF_SUCCESS) + return (CRYPTO_MECHANISM_INVALID); + + /* + * Get the software provider for this mechanism. + * Lock the mech_entry until we grab the 'pd'. + */ + mutex_enter(&me->me_mutex); + + if (me->me_sw_prov == NULL || + (*pd = me->me_sw_prov->pm_prov_desc) == NULL) { + /* no SW provider for this mechanism */ + if (log_warn) + cmn_err(CE_WARN, "no SW provider for \"%s\"\n", + me->me_name); + mutex_exit(&me->me_mutex); + return (CRYPTO_MECH_NOT_SUPPORTED); + } + + KCF_PROV_REFHOLD(*pd); + mutex_exit(&me->me_mutex); + + if (mep != NULL) + *mep = me; + + return (CRYPTO_SUCCESS); +} diff --git a/module/icp/core/kcf_sched.c b/module/icp/core/kcf_sched.c new file mode 100644 index 000000000000..8102d6675fdf --- /dev/null +++ b/module/icp/core/kcf_sched.c @@ -0,0 +1,1763 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * This file contains the core framework routines for the + * kernel cryptographic framework. These routines are at the + * layer, between the kernel API/ioctls and the SPI. + */ + +#include +#include +#include +#include +#include + +kcf_global_swq_t *gswq; /* Global software queue */ + +/* Thread pool related variables */ +static kcf_pool_t *kcfpool; /* Thread pool of kcfd LWPs */ +int kcf_maxthreads = 2; +int kcf_minthreads = 1; +int kcf_thr_multiple = 2; /* Boot-time tunable for experimentation */ +static ulong_t kcf_idlethr_timeout; +#define KCF_DEFAULT_THRTIMEOUT 60000000 /* 60 seconds */ + +/* kmem caches used by the scheduler */ +static kmem_cache_t *kcf_sreq_cache; +static kmem_cache_t *kcf_areq_cache; +static kmem_cache_t *kcf_context_cache; + +/* Global request ID table */ +static kcf_reqid_table_t *kcf_reqid_table[REQID_TABLES]; + +/* KCF stats. Not protected. */ +static kcf_stats_t kcf_ksdata = { + { "total threads in pool", KSTAT_DATA_UINT32}, + { "idle threads in pool", KSTAT_DATA_UINT32}, + { "min threads in pool", KSTAT_DATA_UINT32}, + { "max threads in pool", KSTAT_DATA_UINT32}, + { "requests in gswq", KSTAT_DATA_UINT32}, + { "max requests in gswq", KSTAT_DATA_UINT32}, + { "threads for HW taskq", KSTAT_DATA_UINT32}, + { "minalloc for HW taskq", KSTAT_DATA_UINT32}, + { "maxalloc for HW taskq", KSTAT_DATA_UINT32} +}; + +static kstat_t *kcf_misc_kstat = NULL; +ulong_t kcf_swprov_hndl = 0; + +static kcf_areq_node_t *kcf_areqnode_alloc(kcf_provider_desc_t *, + kcf_context_t *, crypto_call_req_t *, kcf_req_params_t *, boolean_t); +static int kcf_disp_sw_request(kcf_areq_node_t *); +static void process_req_hwp(void *); +static int kcf_enqueue(kcf_areq_node_t *); +static void kcfpool_alloc(void); +static void kcf_reqid_delete(kcf_areq_node_t *areq); +static crypto_req_id_t kcf_reqid_insert(kcf_areq_node_t *areq); +static int kcf_misc_kstat_update(kstat_t *ksp, int rw); + +/* + * Create a new context. + */ +crypto_ctx_t * +kcf_new_ctx(crypto_call_req_t *crq, kcf_provider_desc_t *pd, + crypto_session_id_t sid) +{ + crypto_ctx_t *ctx; + kcf_context_t *kcf_ctx; + + kcf_ctx = kmem_cache_alloc(kcf_context_cache, + (crq == NULL) ? KM_SLEEP : KM_NOSLEEP); + if (kcf_ctx == NULL) + return (NULL); + + /* initialize the context for the consumer */ + kcf_ctx->kc_refcnt = 1; + kcf_ctx->kc_req_chain_first = NULL; + kcf_ctx->kc_req_chain_last = NULL; + kcf_ctx->kc_secondctx = NULL; + KCF_PROV_REFHOLD(pd); + kcf_ctx->kc_prov_desc = pd; + kcf_ctx->kc_sw_prov_desc = NULL; + kcf_ctx->kc_mech = NULL; + + ctx = &kcf_ctx->kc_glbl_ctx; + ctx->cc_provider = pd->pd_prov_handle; + ctx->cc_session = sid; + ctx->cc_provider_private = NULL; + ctx->cc_framework_private = (void *)kcf_ctx; + ctx->cc_flags = 0; + ctx->cc_opstate = NULL; + + return (ctx); +} + +/* + * Allocate a new async request node. + * + * ictx - Framework private context pointer + * crq - Has callback function and argument. Should be non NULL. + * req - The parameters to pass to the SPI + */ +static kcf_areq_node_t * +kcf_areqnode_alloc(kcf_provider_desc_t *pd, kcf_context_t *ictx, + crypto_call_req_t *crq, kcf_req_params_t *req, boolean_t isdual) +{ + kcf_areq_node_t *arptr, *areq; + + ASSERT(crq != NULL); + arptr = kmem_cache_alloc(kcf_areq_cache, KM_NOSLEEP); + if (arptr == NULL) + return (NULL); + + arptr->an_state = REQ_ALLOCATED; + arptr->an_reqarg = *crq; + arptr->an_params = *req; + arptr->an_context = ictx; + arptr->an_isdual = isdual; + + arptr->an_next = arptr->an_prev = NULL; + KCF_PROV_REFHOLD(pd); + arptr->an_provider = pd; + arptr->an_tried_plist = NULL; + arptr->an_refcnt = 1; + arptr->an_idnext = arptr->an_idprev = NULL; + + /* + * Requests for context-less operations do not use the + * fields - an_is_my_turn, and an_ctxchain_next. + */ + if (ictx == NULL) + return (arptr); + + KCF_CONTEXT_REFHOLD(ictx); + /* + * Chain this request to the context. + */ + mutex_enter(&ictx->kc_in_use_lock); + arptr->an_ctxchain_next = NULL; + if ((areq = ictx->kc_req_chain_last) == NULL) { + arptr->an_is_my_turn = B_TRUE; + ictx->kc_req_chain_last = + ictx->kc_req_chain_first = arptr; + } else { + ASSERT(ictx->kc_req_chain_first != NULL); + arptr->an_is_my_turn = B_FALSE; + /* Insert the new request to the end of the chain. */ + areq->an_ctxchain_next = arptr; + ictx->kc_req_chain_last = arptr; + } + mutex_exit(&ictx->kc_in_use_lock); + + return (arptr); +} + +/* + * Queue the request node and do one of the following: + * - If there is an idle thread signal it to run. + * - If there is no idle thread and max running threads is not + * reached, signal the creator thread for more threads. + * + * If the two conditions above are not met, we don't need to do + * any thing. The request will be picked up by one of the + * worker threads when it becomes available. + */ +static int +kcf_disp_sw_request(kcf_areq_node_t *areq) +{ + int err; + int cnt = 0; + + if ((err = kcf_enqueue(areq)) != 0) + return (err); + + if (kcfpool->kp_idlethreads > 0) { + /* Signal an idle thread to run */ + mutex_enter(&gswq->gs_lock); + cv_signal(&gswq->gs_cv); + mutex_exit(&gswq->gs_lock); + + return (CRYPTO_QUEUED); + } + + /* + * We keep the number of running threads to be at + * kcf_minthreads to reduce gs_lock contention. + */ + cnt = kcf_minthreads - + (kcfpool->kp_threads - kcfpool->kp_blockedthreads); + if (cnt > 0) { + /* + * The following ensures the number of threads in pool + * does not exceed kcf_maxthreads. + */ + cnt = MIN(cnt, kcf_maxthreads - (int)kcfpool->kp_threads); + if (cnt > 0) { + /* Signal the creator thread for more threads */ + mutex_enter(&kcfpool->kp_user_lock); + if (!kcfpool->kp_signal_create_thread) { + kcfpool->kp_signal_create_thread = B_TRUE; + kcfpool->kp_nthrs = cnt; + cv_signal(&kcfpool->kp_user_cv); + } + mutex_exit(&kcfpool->kp_user_lock); + } + } + + return (CRYPTO_QUEUED); +} + +/* + * This routine is called by the taskq associated with + * each hardware provider. We notify the kernel consumer + * via the callback routine in case of CRYPTO_SUCCESS or + * a failure. + * + * A request can be of type kcf_areq_node_t or of type + * kcf_sreq_node_t. + */ +static void +process_req_hwp(void *ireq) +{ + int error = 0; + crypto_ctx_t *ctx; + kcf_call_type_t ctype; + kcf_provider_desc_t *pd; + kcf_areq_node_t *areq = (kcf_areq_node_t *)ireq; + kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)ireq; + + pd = ((ctype = GET_REQ_TYPE(ireq)) == CRYPTO_SYNCH) ? + sreq->sn_provider : areq->an_provider; + + /* + * Wait if flow control is in effect for the provider. A + * CRYPTO_PROVIDER_READY or CRYPTO_PROVIDER_FAILED + * notification will signal us. We also get signaled if + * the provider is unregistering. + */ + if (pd->pd_state == KCF_PROV_BUSY) { + mutex_enter(&pd->pd_lock); + while (pd->pd_state == KCF_PROV_BUSY) + cv_wait(&pd->pd_resume_cv, &pd->pd_lock); + mutex_exit(&pd->pd_lock); + } + + /* + * Bump the internal reference count while the request is being + * processed. This is how we know when it's safe to unregister + * a provider. This step must precede the pd_state check below. + */ + KCF_PROV_IREFHOLD(pd); + + /* + * Fail the request if the provider has failed. We return a + * recoverable error and the notified clients attempt any + * recovery. For async clients this is done in kcf_aop_done() + * and for sync clients it is done in the k-api routines. + */ + if (pd->pd_state >= KCF_PROV_FAILED) { + error = CRYPTO_DEVICE_ERROR; + goto bail; + } + + if (ctype == CRYPTO_SYNCH) { + mutex_enter(&sreq->sn_lock); + sreq->sn_state = REQ_INPROGRESS; + mutex_exit(&sreq->sn_lock); + + ctx = sreq->sn_context ? &sreq->sn_context->kc_glbl_ctx : NULL; + error = common_submit_request(sreq->sn_provider, ctx, + sreq->sn_params, sreq); + } else { + kcf_context_t *ictx; + ASSERT(ctype == CRYPTO_ASYNCH); + + /* + * We are in the per-hardware provider thread context and + * hence can sleep. Note that the caller would have done + * a taskq_dispatch(..., TQ_NOSLEEP) and would have returned. + */ + ctx = (ictx = areq->an_context) ? &ictx->kc_glbl_ctx : NULL; + + mutex_enter(&areq->an_lock); + /* + * We need to maintain ordering for multi-part requests. + * an_is_my_turn is set to B_TRUE initially for a request + * when it is enqueued and there are no other requests + * for that context. It is set later from kcf_aop_done() when + * the request before us in the chain of requests for the + * context completes. We get signaled at that point. + */ + if (ictx != NULL) { + ASSERT(ictx->kc_prov_desc == areq->an_provider); + + while (areq->an_is_my_turn == B_FALSE) { + cv_wait(&areq->an_turn_cv, &areq->an_lock); + } + } + areq->an_state = REQ_INPROGRESS; + mutex_exit(&areq->an_lock); + + error = common_submit_request(areq->an_provider, ctx, + &areq->an_params, areq); + } + +bail: + if (error == CRYPTO_QUEUED) { + /* + * The request is queued by the provider and we should + * get a crypto_op_notification() from the provider later. + * We notify the consumer at that time. + */ + return; + } else { /* CRYPTO_SUCCESS or other failure */ + KCF_PROV_IREFRELE(pd); + if (ctype == CRYPTO_SYNCH) + kcf_sop_done(sreq, error); + else + kcf_aop_done(areq, error); + } +} + +/* + * This routine checks if a request can be retried on another + * provider. If true, mech1 is initialized to point to the mechanism + * structure. mech2 is also initialized in case of a dual operation. fg + * is initialized to the correct crypto_func_group_t bit flag. They are + * initialized by this routine, so that the caller can pass them to a + * kcf_get_mech_provider() or kcf_get_dual_provider() with no further change. + * + * We check that the request is for a init or atomic routine and that + * it is for one of the operation groups used from k-api . + */ +static boolean_t +can_resubmit(kcf_areq_node_t *areq, crypto_mechanism_t **mech1, + crypto_mechanism_t **mech2, crypto_func_group_t *fg) +{ + kcf_req_params_t *params; + kcf_op_type_t optype; + + params = &areq->an_params; + optype = params->rp_optype; + + if (!(IS_INIT_OP(optype) || IS_ATOMIC_OP(optype))) + return (B_FALSE); + + switch (params->rp_opgrp) { + case KCF_OG_DIGEST: { + kcf_digest_ops_params_t *dops = ¶ms->rp_u.digest_params; + + dops->do_mech.cm_type = dops->do_framework_mechtype; + *mech1 = &dops->do_mech; + *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DIGEST : + CRYPTO_FG_DIGEST_ATOMIC; + break; + } + + case KCF_OG_MAC: { + kcf_mac_ops_params_t *mops = ¶ms->rp_u.mac_params; + + mops->mo_mech.cm_type = mops->mo_framework_mechtype; + *mech1 = &mops->mo_mech; + *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC : + CRYPTO_FG_MAC_ATOMIC; + break; + } + + case KCF_OG_SIGN: { + kcf_sign_ops_params_t *sops = ¶ms->rp_u.sign_params; + + sops->so_mech.cm_type = sops->so_framework_mechtype; + *mech1 = &sops->so_mech; + switch (optype) { + case KCF_OP_INIT: + *fg = CRYPTO_FG_SIGN; + break; + case KCF_OP_ATOMIC: + *fg = CRYPTO_FG_SIGN_ATOMIC; + break; + default: + ASSERT(optype == KCF_OP_SIGN_RECOVER_ATOMIC); + *fg = CRYPTO_FG_SIGN_RECOVER_ATOMIC; + } + break; + } + + case KCF_OG_VERIFY: { + kcf_verify_ops_params_t *vops = ¶ms->rp_u.verify_params; + + vops->vo_mech.cm_type = vops->vo_framework_mechtype; + *mech1 = &vops->vo_mech; + switch (optype) { + case KCF_OP_INIT: + *fg = CRYPTO_FG_VERIFY; + break; + case KCF_OP_ATOMIC: + *fg = CRYPTO_FG_VERIFY_ATOMIC; + break; + default: + ASSERT(optype == KCF_OP_VERIFY_RECOVER_ATOMIC); + *fg = CRYPTO_FG_VERIFY_RECOVER_ATOMIC; + } + break; + } + + case KCF_OG_ENCRYPT: { + kcf_encrypt_ops_params_t *eops = ¶ms->rp_u.encrypt_params; + + eops->eo_mech.cm_type = eops->eo_framework_mechtype; + *mech1 = &eops->eo_mech; + *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT : + CRYPTO_FG_ENCRYPT_ATOMIC; + break; + } + + case KCF_OG_DECRYPT: { + kcf_decrypt_ops_params_t *dcrops = ¶ms->rp_u.decrypt_params; + + dcrops->dop_mech.cm_type = dcrops->dop_framework_mechtype; + *mech1 = &dcrops->dop_mech; + *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_DECRYPT : + CRYPTO_FG_DECRYPT_ATOMIC; + break; + } + + case KCF_OG_ENCRYPT_MAC: { + kcf_encrypt_mac_ops_params_t *eops = + ¶ms->rp_u.encrypt_mac_params; + + eops->em_encr_mech.cm_type = eops->em_framework_encr_mechtype; + *mech1 = &eops->em_encr_mech; + eops->em_mac_mech.cm_type = eops->em_framework_mac_mechtype; + *mech2 = &eops->em_mac_mech; + *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_ENCRYPT_MAC : + CRYPTO_FG_ENCRYPT_MAC_ATOMIC; + break; + } + + case KCF_OG_MAC_DECRYPT: { + kcf_mac_decrypt_ops_params_t *dops = + ¶ms->rp_u.mac_decrypt_params; + + dops->md_mac_mech.cm_type = dops->md_framework_mac_mechtype; + *mech1 = &dops->md_mac_mech; + dops->md_decr_mech.cm_type = dops->md_framework_decr_mechtype; + *mech2 = &dops->md_decr_mech; + *fg = (optype == KCF_OP_INIT) ? CRYPTO_FG_MAC_DECRYPT : + CRYPTO_FG_MAC_DECRYPT_ATOMIC; + break; + } + + default: + return (B_FALSE); + } + + return (B_TRUE); +} + +/* + * This routine is called when a request to a provider has failed + * with a recoverable error. This routine tries to find another provider + * and dispatches the request to the new provider, if one is available. + * We reuse the request structure. + * + * A return value of NULL from kcf_get_mech_provider() indicates + * we have tried the last provider. + */ +static int +kcf_resubmit_request(kcf_areq_node_t *areq) +{ + int error = CRYPTO_FAILED; + kcf_context_t *ictx; + kcf_provider_desc_t *old_pd; + kcf_provider_desc_t *new_pd; + crypto_mechanism_t *mech1 = NULL, *mech2 = NULL; + crypto_mech_type_t prov_mt1, prov_mt2; + crypto_func_group_t fg; + + if (!can_resubmit(areq, &mech1, &mech2, &fg)) + return (error); + + old_pd = areq->an_provider; + /* + * Add old_pd to the list of providers already tried. We release + * the hold on old_pd (from the earlier kcf_get_mech_provider()) in + * kcf_free_triedlist(). + */ + if (kcf_insert_triedlist(&areq->an_tried_plist, old_pd, + KM_NOSLEEP) == NULL) + return (error); + + if (mech1 && !mech2) { + new_pd = kcf_get_mech_provider(mech1->cm_type, NULL, &error, + areq->an_tried_plist, fg, + (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), 0); + } else { + ASSERT(mech1 != NULL && mech2 != NULL); + + new_pd = kcf_get_dual_provider(mech1, mech2, NULL, &prov_mt1, + &prov_mt2, &error, areq->an_tried_plist, fg, fg, + (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), 0); + } + + if (new_pd == NULL) + return (error); + + /* + * We reuse the old context by resetting provider specific + * fields in it. + */ + if ((ictx = areq->an_context) != NULL) { + crypto_ctx_t *ctx; + + ASSERT(old_pd == ictx->kc_prov_desc); + KCF_PROV_REFRELE(ictx->kc_prov_desc); + KCF_PROV_REFHOLD(new_pd); + ictx->kc_prov_desc = new_pd; + + ctx = &ictx->kc_glbl_ctx; + ctx->cc_provider = new_pd->pd_prov_handle; + ctx->cc_session = new_pd->pd_sid; + ctx->cc_provider_private = NULL; + } + + /* We reuse areq. by resetting the provider and context fields. */ + KCF_PROV_REFRELE(old_pd); + KCF_PROV_REFHOLD(new_pd); + areq->an_provider = new_pd; + mutex_enter(&areq->an_lock); + areq->an_state = REQ_WAITING; + mutex_exit(&areq->an_lock); + + switch (new_pd->pd_prov_type) { + case CRYPTO_SW_PROVIDER: + error = kcf_disp_sw_request(areq); + break; + + case CRYPTO_HW_PROVIDER: { + taskq_t *taskq = new_pd->pd_sched_info.ks_taskq; + + if (taskq_dispatch(taskq, process_req_hwp, areq, TQ_NOSLEEP) == + (taskqid_t)0) { + error = CRYPTO_HOST_MEMORY; + } else { + error = CRYPTO_QUEUED; + } + + break; + default: + break; + } + } + + return (error); +} + +static inline int EMPTY_TASKQ(taskq_t *tq) +{ +#ifdef _KERNEL + return (tq->tq_lowest_id == tq->tq_next_id); +#else + return (tq->tq_task.tqent_next == &tq->tq_task || tq->tq_active == 0); +#endif +} + +/* + * Routine called by both ioctl and k-api. The consumer should + * bundle the parameters into a kcf_req_params_t structure. A bunch + * of macros are available in ops_impl.h for this bundling. They are: + * + * KCF_WRAP_DIGEST_OPS_PARAMS() + * KCF_WRAP_MAC_OPS_PARAMS() + * KCF_WRAP_ENCRYPT_OPS_PARAMS() + * KCF_WRAP_DECRYPT_OPS_PARAMS() ... etc. + * + * It is the caller's responsibility to free the ctx argument when + * appropriate. See the KCF_CONTEXT_COND_RELEASE macro for details. + */ +int +kcf_submit_request(kcf_provider_desc_t *pd, crypto_ctx_t *ctx, + crypto_call_req_t *crq, kcf_req_params_t *params, boolean_t cont) +{ + int error = CRYPTO_SUCCESS; + kcf_areq_node_t *areq; + kcf_sreq_node_t *sreq; + kcf_context_t *kcf_ctx; + taskq_t *taskq = pd->pd_sched_info.ks_taskq; + + kcf_ctx = ctx ? (kcf_context_t *)ctx->cc_framework_private : NULL; + + /* Synchronous cases */ + if (crq == NULL) { + switch (pd->pd_prov_type) { + case CRYPTO_SW_PROVIDER: + error = common_submit_request(pd, ctx, params, + KCF_RHNDL(KM_SLEEP)); + break; + + case CRYPTO_HW_PROVIDER: + /* + * Special case for CRYPTO_SYNCHRONOUS providers that + * never return a CRYPTO_QUEUED error. We skip any + * request allocation and call the SPI directly. + */ + if ((pd->pd_flags & CRYPTO_SYNCHRONOUS) && + EMPTY_TASKQ(taskq)) { + KCF_PROV_IREFHOLD(pd); + if (pd->pd_state == KCF_PROV_READY) { + error = common_submit_request(pd, ctx, + params, KCF_RHNDL(KM_SLEEP)); + KCF_PROV_IREFRELE(pd); + ASSERT(error != CRYPTO_QUEUED); + break; + } + KCF_PROV_IREFRELE(pd); + } + + sreq = kmem_cache_alloc(kcf_sreq_cache, KM_SLEEP); + sreq->sn_state = REQ_ALLOCATED; + sreq->sn_rv = CRYPTO_FAILED; + sreq->sn_params = params; + + /* + * Note that we do not need to hold the context + * for synchronous case as the context will never + * become invalid underneath us. We do not need to hold + * the provider here either as the caller has a hold. + */ + sreq->sn_context = kcf_ctx; + ASSERT(KCF_PROV_REFHELD(pd)); + sreq->sn_provider = pd; + + ASSERT(taskq != NULL); + /* + * Call the SPI directly if the taskq is empty and the + * provider is not busy, else dispatch to the taskq. + * Calling directly is fine as this is the synchronous + * case. This is unlike the asynchronous case where we + * must always dispatch to the taskq. + */ + if (EMPTY_TASKQ(taskq) && + pd->pd_state == KCF_PROV_READY) { + process_req_hwp(sreq); + } else { + /* + * We can not tell from taskq_dispatch() return + * value if we exceeded maxalloc. Hence the + * check here. Since we are allowed to wait in + * the synchronous case, we wait for the taskq + * to become empty. + */ + if (taskq->tq_nalloc >= crypto_taskq_maxalloc) { + taskq_wait(taskq); + } + + (void) taskq_dispatch(taskq, process_req_hwp, + sreq, TQ_SLEEP); + } + + /* + * Wait for the notification to arrive, + * if the operation is not done yet. + * Bug# 4722589 will make the wait a cv_wait_sig(). + */ + mutex_enter(&sreq->sn_lock); + while (sreq->sn_state < REQ_DONE) + cv_wait(&sreq->sn_cv, &sreq->sn_lock); + mutex_exit(&sreq->sn_lock); + + error = sreq->sn_rv; + kmem_cache_free(kcf_sreq_cache, sreq); + + break; + + default: + error = CRYPTO_FAILED; + break; + } + + } else { /* Asynchronous cases */ + switch (pd->pd_prov_type) { + case CRYPTO_SW_PROVIDER: + if (!(crq->cr_flag & CRYPTO_ALWAYS_QUEUE)) { + /* + * This case has less overhead since there is + * no switching of context. + */ + error = common_submit_request(pd, ctx, params, + KCF_RHNDL(KM_NOSLEEP)); + } else { + /* + * CRYPTO_ALWAYS_QUEUE is set. We need to + * queue the request and return. + */ + areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, + params, cont); + if (areq == NULL) + error = CRYPTO_HOST_MEMORY; + else { + if (!(crq->cr_flag + & CRYPTO_SKIP_REQID)) { + /* + * Set the request handle. This handle + * is used for any crypto_cancel_req(9f) + * calls from the consumer. We have to + * do this before dispatching the + * request. + */ + crq->cr_reqid = kcf_reqid_insert(areq); + } + + error = kcf_disp_sw_request(areq); + /* + * There is an error processing this + * request. Remove the handle and + * release the request structure. + */ + if (error != CRYPTO_QUEUED) { + if (!(crq->cr_flag + & CRYPTO_SKIP_REQID)) + kcf_reqid_delete(areq); + KCF_AREQ_REFRELE(areq); + } + } + } + break; + + case CRYPTO_HW_PROVIDER: + /* + * We need to queue the request and return. + */ + areq = kcf_areqnode_alloc(pd, kcf_ctx, crq, params, + cont); + if (areq == NULL) { + error = CRYPTO_HOST_MEMORY; + goto done; + } + + ASSERT(taskq != NULL); + /* + * We can not tell from taskq_dispatch() return + * value if we exceeded maxalloc. Hence the check + * here. + */ + if (taskq->tq_nalloc >= crypto_taskq_maxalloc) { + error = CRYPTO_BUSY; + KCF_AREQ_REFRELE(areq); + goto done; + } + + if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) { + /* + * Set the request handle. This handle is used + * for any crypto_cancel_req(9f) calls from the + * consumer. We have to do this before dispatching + * the request. + */ + crq->cr_reqid = kcf_reqid_insert(areq); + } + + if (taskq_dispatch(taskq, + process_req_hwp, areq, TQ_NOSLEEP) == + (taskqid_t)0) { + error = CRYPTO_HOST_MEMORY; + if (!(crq->cr_flag & CRYPTO_SKIP_REQID)) + kcf_reqid_delete(areq); + KCF_AREQ_REFRELE(areq); + } else { + error = CRYPTO_QUEUED; + } + break; + + default: + error = CRYPTO_FAILED; + break; + } + } + +done: + return (error); +} + +/* + * We're done with this framework context, so free it. Note that freeing + * framework context (kcf_context) frees the global context (crypto_ctx). + * + * The provider is responsible for freeing provider private context after a + * final or single operation and resetting the cc_provider_private field + * to NULL. It should do this before it notifies the framework of the + * completion. We still need to call KCF_PROV_FREE_CONTEXT to handle cases + * like crypto_cancel_ctx(9f). + */ +void +kcf_free_context(kcf_context_t *kcf_ctx) +{ + kcf_provider_desc_t *pd = kcf_ctx->kc_prov_desc; + crypto_ctx_t *gctx = &kcf_ctx->kc_glbl_ctx; + kcf_context_t *kcf_secondctx = kcf_ctx->kc_secondctx; + + /* Release the second context, if any */ + + if (kcf_secondctx != NULL) + KCF_CONTEXT_REFRELE(kcf_secondctx); + + if (gctx->cc_provider_private != NULL) { + mutex_enter(&pd->pd_lock); + if (!KCF_IS_PROV_REMOVED(pd)) { + /* + * Increment the provider's internal refcnt so it + * doesn't unregister from the framework while + * we're calling the entry point. + */ + KCF_PROV_IREFHOLD(pd); + mutex_exit(&pd->pd_lock); + (void) KCF_PROV_FREE_CONTEXT(pd, gctx); + KCF_PROV_IREFRELE(pd); + } else { + mutex_exit(&pd->pd_lock); + } + } + + /* kcf_ctx->kc_prov_desc has a hold on pd */ + KCF_PROV_REFRELE(kcf_ctx->kc_prov_desc); + + /* check if this context is shared with a software provider */ + if ((gctx->cc_flags & CRYPTO_INIT_OPSTATE) && + kcf_ctx->kc_sw_prov_desc != NULL) { + KCF_PROV_REFRELE(kcf_ctx->kc_sw_prov_desc); + } + + kmem_cache_free(kcf_context_cache, kcf_ctx); +} + +/* + * Free the request after releasing all the holds. + */ +void +kcf_free_req(kcf_areq_node_t *areq) +{ + KCF_PROV_REFRELE(areq->an_provider); + if (areq->an_context != NULL) + KCF_CONTEXT_REFRELE(areq->an_context); + + if (areq->an_tried_plist != NULL) + kcf_free_triedlist(areq->an_tried_plist); + kmem_cache_free(kcf_areq_cache, areq); +} + +/* + * Utility routine to remove a request from the chain of requests + * hanging off a context. + */ +void +kcf_removereq_in_ctxchain(kcf_context_t *ictx, kcf_areq_node_t *areq) +{ + kcf_areq_node_t *cur, *prev; + + /* + * Get context lock, search for areq in the chain and remove it. + */ + ASSERT(ictx != NULL); + mutex_enter(&ictx->kc_in_use_lock); + prev = cur = ictx->kc_req_chain_first; + + while (cur != NULL) { + if (cur == areq) { + if (prev == cur) { + if ((ictx->kc_req_chain_first = + cur->an_ctxchain_next) == NULL) + ictx->kc_req_chain_last = NULL; + } else { + if (cur == ictx->kc_req_chain_last) + ictx->kc_req_chain_last = prev; + prev->an_ctxchain_next = cur->an_ctxchain_next; + } + + break; + } + prev = cur; + cur = cur->an_ctxchain_next; + } + mutex_exit(&ictx->kc_in_use_lock); +} + +/* + * Remove the specified node from the global software queue. + * + * The caller must hold the queue lock and request lock (an_lock). + */ +void +kcf_remove_node(kcf_areq_node_t *node) +{ + kcf_areq_node_t *nextp = node->an_next; + kcf_areq_node_t *prevp = node->an_prev; + + if (nextp != NULL) + nextp->an_prev = prevp; + else + gswq->gs_last = prevp; + + if (prevp != NULL) + prevp->an_next = nextp; + else + gswq->gs_first = nextp; + + node->an_state = REQ_CANCELED; +} + +/* + * Add the request node to the end of the global software queue. + * + * The caller should not hold the queue lock. Returns 0 if the + * request is successfully queued. Returns CRYPTO_BUSY if the limit + * on the number of jobs is exceeded. + */ +static int +kcf_enqueue(kcf_areq_node_t *node) +{ + kcf_areq_node_t *tnode; + + mutex_enter(&gswq->gs_lock); + + if (gswq->gs_njobs >= gswq->gs_maxjobs) { + mutex_exit(&gswq->gs_lock); + return (CRYPTO_BUSY); + } + + if (gswq->gs_last == NULL) { + gswq->gs_first = gswq->gs_last = node; + } else { + ASSERT(gswq->gs_last->an_next == NULL); + tnode = gswq->gs_last; + tnode->an_next = node; + gswq->gs_last = node; + node->an_prev = tnode; + } + + gswq->gs_njobs++; + + /* an_lock not needed here as we hold gs_lock */ + node->an_state = REQ_WAITING; + + mutex_exit(&gswq->gs_lock); + + return (0); +} + +/* + * kmem_cache_alloc constructor for sync request structure. + */ +/* ARGSUSED */ +static int +kcf_sreq_cache_constructor(void *buf, void *cdrarg, int kmflags) +{ + kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf; + + sreq->sn_type = CRYPTO_SYNCH; + cv_init(&sreq->sn_cv, NULL, CV_DEFAULT, NULL); + mutex_init(&sreq->sn_lock, NULL, MUTEX_DEFAULT, NULL); + + return (0); +} + +/* ARGSUSED */ +static void +kcf_sreq_cache_destructor(void *buf, void *cdrarg) +{ + kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)buf; + + mutex_destroy(&sreq->sn_lock); + cv_destroy(&sreq->sn_cv); +} + +/* + * kmem_cache_alloc constructor for async request structure. + */ +/* ARGSUSED */ +static int +kcf_areq_cache_constructor(void *buf, void *cdrarg, int kmflags) +{ + kcf_areq_node_t *areq = (kcf_areq_node_t *)buf; + + areq->an_type = CRYPTO_ASYNCH; + areq->an_refcnt = 0; + mutex_init(&areq->an_lock, NULL, MUTEX_DEFAULT, NULL); + cv_init(&areq->an_done, NULL, CV_DEFAULT, NULL); + cv_init(&areq->an_turn_cv, NULL, CV_DEFAULT, NULL); + + return (0); +} + +/* ARGSUSED */ +static void +kcf_areq_cache_destructor(void *buf, void *cdrarg) +{ + kcf_areq_node_t *areq = (kcf_areq_node_t *)buf; + + ASSERT(areq->an_refcnt == 0); + mutex_destroy(&areq->an_lock); + cv_destroy(&areq->an_done); + cv_destroy(&areq->an_turn_cv); +} + +/* + * kmem_cache_alloc constructor for kcf_context structure. + */ +/* ARGSUSED */ +static int +kcf_context_cache_constructor(void *buf, void *cdrarg, int kmflags) +{ + kcf_context_t *kctx = (kcf_context_t *)buf; + + kctx->kc_refcnt = 0; + mutex_init(&kctx->kc_in_use_lock, NULL, MUTEX_DEFAULT, NULL); + + return (0); +} + +/* ARGSUSED */ +static void +kcf_context_cache_destructor(void *buf, void *cdrarg) +{ + kcf_context_t *kctx = (kcf_context_t *)buf; + + ASSERT(kctx->kc_refcnt == 0); + mutex_destroy(&kctx->kc_in_use_lock); +} + +void +kcf_sched_destroy(void) +{ + int i; + + if (kcf_misc_kstat) + kstat_delete(kcf_misc_kstat); + + if (kcfpool) + kmem_free(kcfpool, sizeof (kcf_pool_t)); + + for (i = 0; i < REQID_TABLES; i++) { + if (kcf_reqid_table[i]) + kmem_free(kcf_reqid_table[i], + sizeof (kcf_reqid_table_t)); + } + + if (gswq) + kmem_free(gswq, sizeof (kcf_global_swq_t)); + + if (kcf_context_cache) + kmem_cache_destroy(kcf_context_cache); + if (kcf_areq_cache) + kmem_cache_destroy(kcf_areq_cache); + if (kcf_sreq_cache) + kmem_cache_destroy(kcf_sreq_cache); +} + +/* + * Creates and initializes all the structures needed by the framework. + */ +void +kcf_sched_init(void) +{ + int i; + kcf_reqid_table_t *rt; + + /* + * Create all the kmem caches needed by the framework. We set the + * align argument to 64, to get a slab aligned to 64-byte as well as + * have the objects (cache_chunksize) to be a 64-byte multiple. + * This helps to avoid false sharing as this is the size of the + * CPU cache line. + */ + kcf_sreq_cache = kmem_cache_create("kcf_sreq_cache", + sizeof (struct kcf_sreq_node), 64, kcf_sreq_cache_constructor, + kcf_sreq_cache_destructor, NULL, NULL, NULL, 0); + + kcf_areq_cache = kmem_cache_create("kcf_areq_cache", + sizeof (struct kcf_areq_node), 64, kcf_areq_cache_constructor, + kcf_areq_cache_destructor, NULL, NULL, NULL, 0); + + kcf_context_cache = kmem_cache_create("kcf_context_cache", + sizeof (struct kcf_context), 64, kcf_context_cache_constructor, + kcf_context_cache_destructor, NULL, NULL, NULL, 0); + + gswq = kmem_alloc(sizeof (kcf_global_swq_t), KM_SLEEP); + + mutex_init(&gswq->gs_lock, NULL, MUTEX_DEFAULT, NULL); + cv_init(&gswq->gs_cv, NULL, CV_DEFAULT, NULL); + gswq->gs_njobs = 0; + gswq->gs_maxjobs = kcf_maxthreads * crypto_taskq_maxalloc; + gswq->gs_first = gswq->gs_last = NULL; + + /* Initialize the global reqid table */ + for (i = 0; i < REQID_TABLES; i++) { + rt = kmem_zalloc(sizeof (kcf_reqid_table_t), KM_SLEEP); + kcf_reqid_table[i] = rt; + mutex_init(&rt->rt_lock, NULL, MUTEX_DEFAULT, NULL); + rt->rt_curid = i; + } + + /* Allocate and initialize the thread pool */ + kcfpool_alloc(); + + /* Initialize the event notification list variables */ + mutex_init(&ntfy_list_lock, NULL, MUTEX_DEFAULT, NULL); + cv_init(&ntfy_list_cv, NULL, CV_DEFAULT, NULL); + + /* Create the kcf kstat */ + kcf_misc_kstat = kstat_create("kcf", 0, "framework_stats", "crypto", + KSTAT_TYPE_NAMED, sizeof (kcf_stats_t) / sizeof (kstat_named_t), + KSTAT_FLAG_VIRTUAL); + + if (kcf_misc_kstat != NULL) { + kcf_misc_kstat->ks_data = &kcf_ksdata; + kcf_misc_kstat->ks_update = kcf_misc_kstat_update; + kstat_install(kcf_misc_kstat); + } +} + +/* + * Signal the waiting sync client. + */ +void +kcf_sop_done(kcf_sreq_node_t *sreq, int error) +{ + mutex_enter(&sreq->sn_lock); + sreq->sn_state = REQ_DONE; + sreq->sn_rv = error; + cv_signal(&sreq->sn_cv); + mutex_exit(&sreq->sn_lock); +} + +/* + * Callback the async client with the operation status. + * We free the async request node and possibly the context. + * We also handle any chain of requests hanging off of + * the context. + */ +void +kcf_aop_done(kcf_areq_node_t *areq, int error) +{ + kcf_op_type_t optype; + boolean_t skip_notify = B_FALSE; + kcf_context_t *ictx; + kcf_areq_node_t *nextreq; + + /* + * Handle recoverable errors. This has to be done first + * before doing any thing else in this routine so that + * we do not change the state of the request. + */ + if (error != CRYPTO_SUCCESS && IS_RECOVERABLE(error)) { + /* + * We try another provider, if one is available. Else + * we continue with the failure notification to the + * client. + */ + if (kcf_resubmit_request(areq) == CRYPTO_QUEUED) + return; + } + + mutex_enter(&areq->an_lock); + areq->an_state = REQ_DONE; + mutex_exit(&areq->an_lock); + + optype = (&areq->an_params)->rp_optype; + if ((ictx = areq->an_context) != NULL) { + /* + * A request after it is removed from the request + * queue, still stays on a chain of requests hanging + * of its context structure. It needs to be removed + * from this chain at this point. + */ + mutex_enter(&ictx->kc_in_use_lock); + nextreq = areq->an_ctxchain_next; + if (nextreq != NULL) { + mutex_enter(&nextreq->an_lock); + nextreq->an_is_my_turn = B_TRUE; + cv_signal(&nextreq->an_turn_cv); + mutex_exit(&nextreq->an_lock); + } + + ictx->kc_req_chain_first = nextreq; + if (nextreq == NULL) + ictx->kc_req_chain_last = NULL; + mutex_exit(&ictx->kc_in_use_lock); + + if (IS_SINGLE_OP(optype) || IS_FINAL_OP(optype)) { + ASSERT(nextreq == NULL); + KCF_CONTEXT_REFRELE(ictx); + } else if (error != CRYPTO_SUCCESS && IS_INIT_OP(optype)) { + /* + * NOTE - We do not release the context in case of update + * operations. We require the consumer to free it explicitly, + * in case it wants to abandon an update operation. This is done + * as there may be mechanisms in ECB mode that can continue + * even if an operation on a block fails. + */ + KCF_CONTEXT_REFRELE(ictx); + } + } + + /* Deal with the internal continuation to this request first */ + + if (areq->an_isdual) { + kcf_dual_req_t *next_arg; + next_arg = (kcf_dual_req_t *)areq->an_reqarg.cr_callback_arg; + next_arg->kr_areq = areq; + KCF_AREQ_REFHOLD(areq); + areq->an_isdual = B_FALSE; + + NOTIFY_CLIENT(areq, error); + return; + } + + /* + * If CRYPTO_NOTIFY_OPDONE flag is set, we should notify + * always. If this flag is clear, we skip the notification + * provided there are no errors. We check this flag for only + * init or update operations. It is ignored for single, final or + * atomic operations. + */ + skip_notify = (IS_UPDATE_OP(optype) || IS_INIT_OP(optype)) && + (!(areq->an_reqarg.cr_flag & CRYPTO_NOTIFY_OPDONE)) && + (error == CRYPTO_SUCCESS); + + if (!skip_notify) { + NOTIFY_CLIENT(areq, error); + } + + if (!(areq->an_reqarg.cr_flag & CRYPTO_SKIP_REQID)) + kcf_reqid_delete(areq); + + KCF_AREQ_REFRELE(areq); +} + +/* + * Allocate the thread pool and initialize all the fields. + */ +static void +kcfpool_alloc() +{ + kcfpool = kmem_alloc(sizeof (kcf_pool_t), KM_SLEEP); + + kcfpool->kp_threads = kcfpool->kp_idlethreads = 0; + kcfpool->kp_blockedthreads = 0; + kcfpool->kp_signal_create_thread = B_FALSE; + kcfpool->kp_nthrs = 0; + kcfpool->kp_user_waiting = B_FALSE; + + mutex_init(&kcfpool->kp_thread_lock, NULL, MUTEX_DEFAULT, NULL); + cv_init(&kcfpool->kp_nothr_cv, NULL, CV_DEFAULT, NULL); + + mutex_init(&kcfpool->kp_user_lock, NULL, MUTEX_DEFAULT, NULL); + cv_init(&kcfpool->kp_user_cv, NULL, CV_DEFAULT, NULL); + + kcf_idlethr_timeout = KCF_DEFAULT_THRTIMEOUT; +} + +/* + * Insert the async request in the hash table after assigning it + * an ID. Returns the ID. + * + * The ID is used by the caller to pass as an argument to a + * cancel_req() routine later. + */ +static crypto_req_id_t +kcf_reqid_insert(kcf_areq_node_t *areq) +{ + int indx; + crypto_req_id_t id; + kcf_areq_node_t *headp; + kcf_reqid_table_t *rt = + kcf_reqid_table[CPU_SEQID & REQID_TABLE_MASK]; + + mutex_enter(&rt->rt_lock); + + rt->rt_curid = id = + (rt->rt_curid - REQID_COUNTER_LOW) | REQID_COUNTER_HIGH; + SET_REQID(areq, id); + indx = REQID_HASH(id); + headp = areq->an_idnext = rt->rt_idhash[indx]; + areq->an_idprev = NULL; + if (headp != NULL) + headp->an_idprev = areq; + + rt->rt_idhash[indx] = areq; + mutex_exit(&rt->rt_lock); + + return (id); +} + +/* + * Delete the async request from the hash table. + */ +static void +kcf_reqid_delete(kcf_areq_node_t *areq) +{ + int indx; + kcf_areq_node_t *nextp, *prevp; + crypto_req_id_t id = GET_REQID(areq); + kcf_reqid_table_t *rt; + + rt = kcf_reqid_table[id & REQID_TABLE_MASK]; + indx = REQID_HASH(id); + + mutex_enter(&rt->rt_lock); + + nextp = areq->an_idnext; + prevp = areq->an_idprev; + if (nextp != NULL) + nextp->an_idprev = prevp; + if (prevp != NULL) + prevp->an_idnext = nextp; + else + rt->rt_idhash[indx] = nextp; + + SET_REQID(areq, 0); + cv_broadcast(&areq->an_done); + + mutex_exit(&rt->rt_lock); +} + +/* + * Cancel a single asynchronous request. + * + * We guarantee that no problems will result from calling + * crypto_cancel_req() for a request which is either running, or + * has already completed. We remove the request from any queues + * if it is possible. We wait for request completion if the + * request is dispatched to a provider. + * + * Calling context: + * Can be called from user context only. + * + * NOTE: We acquire the following locks in this routine (in order): + * - rt_lock (kcf_reqid_table_t) + * - gswq->gs_lock + * - areq->an_lock + * - ictx->kc_in_use_lock (from kcf_removereq_in_ctxchain()) + * + * This locking order MUST be maintained in code every where else. + */ +void +crypto_cancel_req(crypto_req_id_t id) +{ + int indx; + kcf_areq_node_t *areq; + kcf_provider_desc_t *pd; + kcf_context_t *ictx; + kcf_reqid_table_t *rt; + + rt = kcf_reqid_table[id & REQID_TABLE_MASK]; + indx = REQID_HASH(id); + + mutex_enter(&rt->rt_lock); + for (areq = rt->rt_idhash[indx]; areq; areq = areq->an_idnext) { + if (GET_REQID(areq) == id) { + /* + * We found the request. It is either still waiting + * in the framework queues or running at the provider. + */ + pd = areq->an_provider; + ASSERT(pd != NULL); + + switch (pd->pd_prov_type) { + case CRYPTO_SW_PROVIDER: + mutex_enter(&gswq->gs_lock); + mutex_enter(&areq->an_lock); + + /* This request can be safely canceled. */ + if (areq->an_state <= REQ_WAITING) { + /* Remove from gswq, global software queue. */ + kcf_remove_node(areq); + if ((ictx = areq->an_context) != NULL) + kcf_removereq_in_ctxchain(ictx, areq); + + mutex_exit(&areq->an_lock); + mutex_exit(&gswq->gs_lock); + mutex_exit(&rt->rt_lock); + + /* Remove areq from hash table and free it. */ + kcf_reqid_delete(areq); + KCF_AREQ_REFRELE(areq); + return; + } + + mutex_exit(&areq->an_lock); + mutex_exit(&gswq->gs_lock); + break; + + case CRYPTO_HW_PROVIDER: + /* + * There is no interface to remove an entry + * once it is on the taskq. So, we do not do + * any thing for a hardware provider. + */ + break; + default: + break; + } + + /* + * The request is running. Wait for the request completion + * to notify us. + */ + KCF_AREQ_REFHOLD(areq); + while (GET_REQID(areq) == id) + cv_wait(&areq->an_done, &rt->rt_lock); + KCF_AREQ_REFRELE(areq); + break; + } + } + + mutex_exit(&rt->rt_lock); +} + +/* + * Cancel all asynchronous requests associated with the + * passed in crypto context and free it. + * + * A client SHOULD NOT call this routine after calling a crypto_*_final + * routine. This routine is called only during intermediate operations. + * The client should not use the crypto context after this function returns + * since we destroy it. + * + * Calling context: + * Can be called from user context only. + */ +void +crypto_cancel_ctx(crypto_context_t ctx) +{ + kcf_context_t *ictx; + kcf_areq_node_t *areq; + + if (ctx == NULL) + return; + + ictx = (kcf_context_t *)((crypto_ctx_t *)ctx)->cc_framework_private; + + mutex_enter(&ictx->kc_in_use_lock); + + /* Walk the chain and cancel each request */ + while ((areq = ictx->kc_req_chain_first) != NULL) { + /* + * We have to drop the lock here as we may have + * to wait for request completion. We hold the + * request before dropping the lock though, so that it + * won't be freed underneath us. + */ + KCF_AREQ_REFHOLD(areq); + mutex_exit(&ictx->kc_in_use_lock); + + crypto_cancel_req(GET_REQID(areq)); + KCF_AREQ_REFRELE(areq); + + mutex_enter(&ictx->kc_in_use_lock); + } + + mutex_exit(&ictx->kc_in_use_lock); + KCF_CONTEXT_REFRELE(ictx); +} + +/* + * Update kstats. + */ +static int +kcf_misc_kstat_update(kstat_t *ksp, int rw) +{ + uint_t tcnt; + kcf_stats_t *ks_data; + + if (rw == KSTAT_WRITE) + return (EACCES); + + ks_data = ksp->ks_data; + + ks_data->ks_thrs_in_pool.value.ui32 = kcfpool->kp_threads; + /* + * The failover thread is counted in kp_idlethreads in + * some corner cases. This is done to avoid doing more checks + * when submitting a request. We account for those cases below. + */ + if ((tcnt = kcfpool->kp_idlethreads) == (kcfpool->kp_threads + 1)) + tcnt--; + ks_data->ks_idle_thrs.value.ui32 = tcnt; + ks_data->ks_minthrs.value.ui32 = kcf_minthreads; + ks_data->ks_maxthrs.value.ui32 = kcf_maxthreads; + ks_data->ks_swq_njobs.value.ui32 = gswq->gs_njobs; + ks_data->ks_swq_maxjobs.value.ui32 = gswq->gs_maxjobs; + ks_data->ks_taskq_threads.value.ui32 = crypto_taskq_threads; + ks_data->ks_taskq_minalloc.value.ui32 = crypto_taskq_minalloc; + ks_data->ks_taskq_maxalloc.value.ui32 = crypto_taskq_maxalloc; + + return (0); +} + +/* + * Allocate and initiatize a kcf_dual_req, used for saving the arguments of + * a dual operation or an atomic operation that has to be internally + * simulated with multiple single steps. + * crq determines the memory allocation flags. + */ + +kcf_dual_req_t * +kcf_alloc_req(crypto_call_req_t *crq) +{ + kcf_dual_req_t *kcr; + + kcr = kmem_alloc(sizeof (kcf_dual_req_t), KCF_KMFLAG(crq)); + + if (kcr == NULL) + return (NULL); + + /* Copy the whole crypto_call_req struct, as it isn't persistant */ + if (crq != NULL) + kcr->kr_callreq = *crq; + else + bzero(&(kcr->kr_callreq), sizeof (crypto_call_req_t)); + kcr->kr_areq = NULL; + kcr->kr_saveoffset = 0; + kcr->kr_savelen = 0; + + return (kcr); +} + +/* + * Callback routine for the next part of a simulated dual part. + * Schedules the next step. + * + * This routine can be called from interrupt context. + */ +void +kcf_next_req(void *next_req_arg, int status) +{ + kcf_dual_req_t *next_req = (kcf_dual_req_t *)next_req_arg; + kcf_req_params_t *params = &(next_req->kr_params); + kcf_areq_node_t *areq = next_req->kr_areq; + int error = status; + kcf_provider_desc_t *pd = NULL; + crypto_dual_data_t *ct = NULL; + + /* Stop the processing if an error occured at this step */ + if (error != CRYPTO_SUCCESS) { +out: + areq->an_reqarg = next_req->kr_callreq; + KCF_AREQ_REFRELE(areq); + kmem_free(next_req, sizeof (kcf_dual_req_t)); + areq->an_isdual = B_FALSE; + kcf_aop_done(areq, error); + return; + } + + switch (params->rp_opgrp) { + case KCF_OG_MAC: { + + /* + * The next req is submitted with the same reqid as the + * first part. The consumer only got back that reqid, and + * should still be able to cancel the operation during its + * second step. + */ + kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params); + crypto_ctx_template_t mac_tmpl; + kcf_mech_entry_t *me; + + ct = (crypto_dual_data_t *)mops->mo_data; + mac_tmpl = (crypto_ctx_template_t)mops->mo_templ; + + /* No expected recoverable failures, so no retry list */ + pd = kcf_get_mech_provider(mops->mo_framework_mechtype, + &me, &error, NULL, CRYPTO_FG_MAC_ATOMIC, + (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), ct->dd_len2); + + if (pd == NULL) { + error = CRYPTO_MECH_NOT_SUPPORTED; + goto out; + } + /* Validate the MAC context template here */ + if ((pd->pd_prov_type == CRYPTO_SW_PROVIDER) && + (mac_tmpl != NULL)) { + kcf_ctx_template_t *ctx_mac_tmpl; + + ctx_mac_tmpl = (kcf_ctx_template_t *)mac_tmpl; + + if (ctx_mac_tmpl->ct_generation != me->me_gen_swprov) { + KCF_PROV_REFRELE(pd); + error = CRYPTO_OLD_CTX_TEMPLATE; + goto out; + } + mops->mo_templ = ctx_mac_tmpl->ct_prov_tmpl; + } + + break; + } + case KCF_OG_DECRYPT: { + kcf_decrypt_ops_params_t *dcrops = + &(params->rp_u.decrypt_params); + + ct = (crypto_dual_data_t *)dcrops->dop_ciphertext; + /* No expected recoverable failures, so no retry list */ + pd = kcf_get_mech_provider(dcrops->dop_framework_mechtype, + NULL, &error, NULL, CRYPTO_FG_DECRYPT_ATOMIC, + (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED), ct->dd_len1); + + if (pd == NULL) { + error = CRYPTO_MECH_NOT_SUPPORTED; + goto out; + } + break; + } + default: + break; + } + + /* The second step uses len2 and offset2 of the dual_data */ + next_req->kr_saveoffset = ct->dd_offset1; + next_req->kr_savelen = ct->dd_len1; + ct->dd_offset1 = ct->dd_offset2; + ct->dd_len1 = ct->dd_len2; + + /* preserve if the caller is restricted */ + if (areq->an_reqarg.cr_flag & CRYPTO_RESTRICTED) { + areq->an_reqarg.cr_flag = CRYPTO_RESTRICTED; + } else { + areq->an_reqarg.cr_flag = 0; + } + + areq->an_reqarg.cr_callback_func = kcf_last_req; + areq->an_reqarg.cr_callback_arg = next_req; + areq->an_isdual = B_TRUE; + + /* + * We would like to call kcf_submit_request() here. But, + * that is not possible as that routine allocates a new + * kcf_areq_node_t request structure, while we need to + * reuse the existing request structure. + */ + switch (pd->pd_prov_type) { + case CRYPTO_SW_PROVIDER: + error = common_submit_request(pd, NULL, params, + KCF_RHNDL(KM_NOSLEEP)); + break; + + case CRYPTO_HW_PROVIDER: { + kcf_provider_desc_t *old_pd; + taskq_t *taskq = pd->pd_sched_info.ks_taskq; + + /* + * Set the params for the second step in the + * dual-ops. + */ + areq->an_params = *params; + old_pd = areq->an_provider; + KCF_PROV_REFRELE(old_pd); + KCF_PROV_REFHOLD(pd); + areq->an_provider = pd; + + /* + * Note that we have to do a taskq_dispatch() + * here as we may be in interrupt context. + */ + if (taskq_dispatch(taskq, process_req_hwp, areq, + TQ_NOSLEEP) == (taskqid_t)0) { + error = CRYPTO_HOST_MEMORY; + } else { + error = CRYPTO_QUEUED; + } + break; + } + default: + break; + } + + /* + * We have to release the holds on the request and the provider + * in all cases. + */ + KCF_AREQ_REFRELE(areq); + KCF_PROV_REFRELE(pd); + + if (error != CRYPTO_QUEUED) { + /* restore, clean up, and invoke the client's callback */ + + ct->dd_offset1 = next_req->kr_saveoffset; + ct->dd_len1 = next_req->kr_savelen; + areq->an_reqarg = next_req->kr_callreq; + kmem_free(next_req, sizeof (kcf_dual_req_t)); + areq->an_isdual = B_FALSE; + kcf_aop_done(areq, error); + } +} + +/* + * Last part of an emulated dual operation. + * Clean up and restore ... + */ +void +kcf_last_req(void *last_req_arg, int status) +{ + kcf_dual_req_t *last_req = (kcf_dual_req_t *)last_req_arg; + + kcf_req_params_t *params = &(last_req->kr_params); + kcf_areq_node_t *areq = last_req->kr_areq; + crypto_dual_data_t *ct = NULL; + + switch (params->rp_opgrp) { + case KCF_OG_MAC: { + kcf_mac_ops_params_t *mops = &(params->rp_u.mac_params); + + ct = (crypto_dual_data_t *)mops->mo_data; + break; + } + case KCF_OG_DECRYPT: { + kcf_decrypt_ops_params_t *dcrops = + &(params->rp_u.decrypt_params); + + ct = (crypto_dual_data_t *)dcrops->dop_ciphertext; + break; + } + default: + break; + } + ct->dd_offset1 = last_req->kr_saveoffset; + ct->dd_len1 = last_req->kr_savelen; + + /* The submitter used kcf_last_req as its callback */ + + if (areq == NULL) { + crypto_call_req_t *cr = &last_req->kr_callreq; + + (*(cr->cr_callback_func))(cr->cr_callback_arg, status); + kmem_free(last_req, sizeof (kcf_dual_req_t)); + return; + } + areq->an_reqarg = last_req->kr_callreq; + KCF_AREQ_REFRELE(areq); + kmem_free(last_req, sizeof (kcf_dual_req_t)); + areq->an_isdual = B_FALSE; + kcf_aop_done(areq, status); +} diff --git a/module/icp/illumos-crypto.c b/module/icp/illumos-crypto.c new file mode 100644 index 000000000000..7dd5dbf42fc4 --- /dev/null +++ b/module/icp/illumos-crypto.c @@ -0,0 +1,152 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2016, Datto, Inc. All rights reserved. + */ + +#ifdef _KERNEL +#include +#include +#include +#else +#define __exit +#define __init +#endif + +#include +#include +#include +#include +#include +#include + +/* + * Changes made to the original Illumos Crypto Layer for the ICP: + * + * Several changes were needed to allow the Illumos Crypto Layer + * to work in the Linux kernel. Almost all of the changes fall into + * one of the following categories: + * + * 1) Moving the syntax to the C90: This was mostly a matter of + * changing func() definitions to func(void). In a few cases, + * initializations of structs with unions needed to have brackets + * added. + * + * 2) Changes to allow userspace compilation: The ICP is meant to be + * compiled and used in both userspace and kernel space (for ztest and + * libzfs), so the _KERNEL macros did not make sense anymore. For the + * same reason, many header includes were also changed to use + * sys/zfs_context.h + * + * 3) Moving to a statically compiled architecture: At some point in + * the future it may make sense to have encryption algorithms that are + * loadable into the ICP at runtime via separate kernel modules. + * However, considering that this code will probably not see much use + * outside of zfs and zfs encryption only requires aes and sha256 + * algorithms it seemed like more trouble than it was worth to port over + * Illumos's kernel module structure to a Linux kernel module. In + * addition, The Illumos code related to keeping track of kernel modules + * is very much tied to the Illumos OS and proved difficult to port to + * Linux. Therefore, the structure of the ICP was simplified to work + * statically and several pieces of code responsible for keeping track + * of Illumos kernel modules were removed and simplified. All module + * initialization and destruction is now called in this file during + * Linux kernel module loading and unloading. + * + * 4) Adding destructors: The Illumos Crypto Layer is built into + * the Illumos kernel and is not meant to be unloaded. Some destructors + * were added to allow the ICP to be unloaded without leaking + * structures. + * + * 5) Removing CRYPTO_DATA_MBLK related structures and code: + * crypto_data_t can have 3 formats, CRYPTO_DATA_RAW, CRYPTO_DATA_UIO, + * and CRYPTO_DATA_MBLK. ZFS only requires the first 2 formats, as the + * last one is related to streamed data. To simplify the port, code + * related to this format was removed. + * + * 6) Changes for architecture specific code: Some changes were needed + * to make architecture specific assembly compile. The biggest change + * here was to functions related to detecting CPU capabilities for amd64. + * The Illumos Crypto Layer used called into the Illumos kernel's API + * to discover these. They have been converted to instead use the + * 'cpuid' instruction as per the Intel spec. In addition, references to + * the sun4u' and sparc architectures have been removed so that these + * will use the generic implementation. + * + * 7) Removing sha384 and sha512 code: The sha code was actually very + * wasy to port. However, the generic sha384 and sha512 code actually + * exceeds the stack size on arm and powerpc architectures. In an effort + * to remove warnings, this code was removed. + * + * 8) Change large allocations from kmem_alloc() to vmem_alloc(): In + * testing the ICP with the ZFS encryption code, a few allocations were + * found that could potentially be very large. These caused the SPL to + * throw warnings and so they were changed to use vmem_alloc(). + * + * 9) Makefiles: Makefiles were added that would work with the existing + * ZFS Makefiles. + */ + +void __exit +icp_fini(void) +{ + sha2_mod_fini(); + sha1_mod_fini(); + aes_mod_fini(); + kcf_sched_destroy(); + kcf_prov_tab_destroy(); + kcf_destroy_mech_tabs(); + mod_hash_fini(); +} + +/* roughly equivalent to kcf.c: _init() */ +int __init +icp_init(void) +{ + /* initialize the mod hash module */ + mod_hash_init(); + + /* initialize the mechanisms tables supported out-of-the-box */ + kcf_init_mech_tabs(); + + /* initialize the providers tables */ + kcf_prov_tab_init(); + + /* + * Initialize scheduling structures. Note that this does NOT + * start any threads since it might not be safe to do so. + */ + kcf_sched_init(); + + /* initialize algorithms */ + aes_mod_init(); + sha1_mod_init(); + sha2_mod_init(); + + return (0); +} + +#if defined(_KERNEL) && defined(HAVE_SPL) +module_exit(icp_fini); +module_init(icp_init); +MODULE_LICENSE(ZFS_META_LICENSE); +#endif diff --git a/module/icp/include/aes/aes_impl.h b/module/icp/include/aes/aes_impl.h new file mode 100644 index 000000000000..ed15f74e797e --- /dev/null +++ b/module/icp/include/aes/aes_impl.h @@ -0,0 +1,170 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _AES_IMPL_H +#define _AES_IMPL_H + +/* + * Common definitions used by AES. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +/* Similar to sysmacros.h IS_P2ALIGNED, but checks two pointers: */ +#define IS_P2ALIGNED2(v, w, a) \ + ((((uintptr_t)(v) | (uintptr_t)(w)) & ((uintptr_t)(a) - 1)) == 0) + +#define AES_BLOCK_LEN 16 /* bytes */ +/* Round constant length, in number of 32-bit elements: */ +#define RC_LENGTH (5 * ((AES_BLOCK_LEN) / 4 - 2)) + +#define AES_COPY_BLOCK(src, dst) \ + (dst)[0] = (src)[0]; \ + (dst)[1] = (src)[1]; \ + (dst)[2] = (src)[2]; \ + (dst)[3] = (src)[3]; \ + (dst)[4] = (src)[4]; \ + (dst)[5] = (src)[5]; \ + (dst)[6] = (src)[6]; \ + (dst)[7] = (src)[7]; \ + (dst)[8] = (src)[8]; \ + (dst)[9] = (src)[9]; \ + (dst)[10] = (src)[10]; \ + (dst)[11] = (src)[11]; \ + (dst)[12] = (src)[12]; \ + (dst)[13] = (src)[13]; \ + (dst)[14] = (src)[14]; \ + (dst)[15] = (src)[15] + +#define AES_XOR_BLOCK(src, dst) \ + (dst)[0] ^= (src)[0]; \ + (dst)[1] ^= (src)[1]; \ + (dst)[2] ^= (src)[2]; \ + (dst)[3] ^= (src)[3]; \ + (dst)[4] ^= (src)[4]; \ + (dst)[5] ^= (src)[5]; \ + (dst)[6] ^= (src)[6]; \ + (dst)[7] ^= (src)[7]; \ + (dst)[8] ^= (src)[8]; \ + (dst)[9] ^= (src)[9]; \ + (dst)[10] ^= (src)[10]; \ + (dst)[11] ^= (src)[11]; \ + (dst)[12] ^= (src)[12]; \ + (dst)[13] ^= (src)[13]; \ + (dst)[14] ^= (src)[14]; \ + (dst)[15] ^= (src)[15] + +/* AES key size definitions */ +#define AES_MINBITS 128 +#define AES_MINBYTES ((AES_MINBITS) >> 3) +#define AES_MAXBITS 256 +#define AES_MAXBYTES ((AES_MAXBITS) >> 3) + +#define AES_MIN_KEY_BYTES ((AES_MINBITS) >> 3) +#define AES_MAX_KEY_BYTES ((AES_MAXBITS) >> 3) +#define AES_192_KEY_BYTES 24 +#define AES_IV_LEN 16 + +/* AES key schedule may be implemented with 32- or 64-bit elements: */ +#define AES_32BIT_KS 32 +#define AES_64BIT_KS 64 + +#define MAX_AES_NR 14 /* Maximum number of rounds */ +#define MAX_AES_NB 4 /* Number of columns comprising a state */ + +typedef union { +#ifdef sun4u + uint64_t ks64[((MAX_AES_NR) + 1) * (MAX_AES_NB)]; +#endif + uint32_t ks32[((MAX_AES_NR) + 1) * (MAX_AES_NB)]; +} aes_ks_t; + +/* aes_key.flags value: */ +#define INTEL_AES_NI_CAPABLE 0x1 /* AES-NI instructions present */ + +typedef struct aes_key aes_key_t; +struct aes_key { + aes_ks_t encr_ks; /* encryption key schedule */ + aes_ks_t decr_ks; /* decryption key schedule */ +#ifdef __amd64 + long double align128; /* Align fields above for Intel AES-NI */ + int flags; /* implementation-dependent flags */ +#endif /* __amd64 */ + int nr; /* number of rounds (10, 12, or 14) */ + int type; /* key schedule size (32 or 64 bits) */ +}; + +/* + * Core AES functions. + * ks and keysched are pointers to aes_key_t. + * They are declared void* as they are intended to be opaque types. + * Use function aes_alloc_keysched() to allocate memory for ks and keysched. + */ +extern void *aes_alloc_keysched(size_t *size, int kmflag); +extern void aes_init_keysched(const uint8_t *cipherKey, uint_t keyBits, + void *keysched); +extern int aes_encrypt_block(const void *ks, const uint8_t *pt, uint8_t *ct); +extern int aes_decrypt_block(const void *ks, const uint8_t *ct, uint8_t *pt); + +/* + * AES mode functions. + * The first 2 functions operate on 16-byte AES blocks. + */ +extern void aes_copy_block(uint8_t *in, uint8_t *out); +extern void aes_xor_block(uint8_t *data, uint8_t *dst); + +/* Note: ctx is a pointer to aes_ctx_t defined in modes.h */ +extern int aes_encrypt_contiguous_blocks(void *ctx, char *data, size_t length, + crypto_data_t *out); +extern int aes_decrypt_contiguous_blocks(void *ctx, char *data, size_t length, + crypto_data_t *out); + +/* + * The following definitions and declarations are only used by AES FIPS POST + */ +#ifdef _AES_IMPL + +typedef enum aes_mech_type { + AES_ECB_MECH_INFO_TYPE, /* SUN_CKM_AES_ECB */ + AES_CBC_MECH_INFO_TYPE, /* SUN_CKM_AES_CBC */ + AES_CBC_PAD_MECH_INFO_TYPE, /* SUN_CKM_AES_CBC_PAD */ + AES_CTR_MECH_INFO_TYPE, /* SUN_CKM_AES_CTR */ + AES_CCM_MECH_INFO_TYPE, /* SUN_CKM_AES_CCM */ + AES_GCM_MECH_INFO_TYPE, /* SUN_CKM_AES_GCM */ + AES_GMAC_MECH_INFO_TYPE /* SUN_CKM_AES_GMAC */ +} aes_mech_type_t; + +#endif /* _AES_IMPL */ + +#ifdef __cplusplus +} +#endif + +#endif /* _AES_IMPL_H */ diff --git a/module/icp/include/modes/modes.h b/module/icp/include/modes/modes.h new file mode 100644 index 000000000000..7c1f10b16e76 --- /dev/null +++ b/module/icp/include/modes/modes.h @@ -0,0 +1,385 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _COMMON_CRYPTO_MODES_H +#define _COMMON_CRYPTO_MODES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#define ECB_MODE 0x00000002 +#define CBC_MODE 0x00000004 +#define CTR_MODE 0x00000008 +#define CCM_MODE 0x00000010 +#define GCM_MODE 0x00000020 +#define GMAC_MODE 0x00000040 + +/* + * cc_keysched: Pointer to key schedule. + * + * cc_keysched_len: Length of the key schedule. + * + * cc_remainder: This is for residual data, i.e. data that can't + * be processed because there are too few bytes. + * Must wait until more data arrives. + * + * cc_remainder_len: Number of bytes in cc_remainder. + * + * cc_iv: Scratch buffer that sometimes contains the IV. + * + * cc_lastp: Pointer to previous block of ciphertext. + * + * cc_copy_to: Pointer to where encrypted residual data needs + * to be copied. + * + * cc_flags: PROVIDER_OWNS_KEY_SCHEDULE + * When a context is freed, it is necessary + * to know whether the key schedule was allocated + * by the caller, or internally, e.g. an init routine. + * If allocated by the latter, then it needs to be freed. + * + * ECB_MODE, CBC_MODE, CTR_MODE, or CCM_MODE + */ +struct common_ctx { + void *cc_keysched; + size_t cc_keysched_len; + uint64_t cc_iv[2]; + uint64_t cc_remainder[2]; + size_t cc_remainder_len; + uint8_t *cc_lastp; + uint8_t *cc_copy_to; + uint32_t cc_flags; +}; + +typedef struct common_ctx common_ctx_t; + +typedef struct ecb_ctx { + struct common_ctx ecb_common; + uint64_t ecb_lastblock[2]; +} ecb_ctx_t; + +#define ecb_keysched ecb_common.cc_keysched +#define ecb_keysched_len ecb_common.cc_keysched_len +#define ecb_iv ecb_common.cc_iv +#define ecb_remainder ecb_common.cc_remainder +#define ecb_remainder_len ecb_common.cc_remainder_len +#define ecb_lastp ecb_common.cc_lastp +#define ecb_copy_to ecb_common.cc_copy_to +#define ecb_flags ecb_common.cc_flags + +typedef struct cbc_ctx { + struct common_ctx cbc_common; + uint64_t cbc_lastblock[2]; +} cbc_ctx_t; + +#define cbc_keysched cbc_common.cc_keysched +#define cbc_keysched_len cbc_common.cc_keysched_len +#define cbc_iv cbc_common.cc_iv +#define cbc_remainder cbc_common.cc_remainder +#define cbc_remainder_len cbc_common.cc_remainder_len +#define cbc_lastp cbc_common.cc_lastp +#define cbc_copy_to cbc_common.cc_copy_to +#define cbc_flags cbc_common.cc_flags + +/* + * ctr_lower_mask Bit-mask for lower 8 bytes of counter block. + * ctr_upper_mask Bit-mask for upper 8 bytes of counter block. + */ +typedef struct ctr_ctx { + struct common_ctx ctr_common; + uint64_t ctr_lower_mask; + uint64_t ctr_upper_mask; + uint32_t ctr_tmp[4]; +} ctr_ctx_t; + +/* + * ctr_cb Counter block. + */ +#define ctr_keysched ctr_common.cc_keysched +#define ctr_keysched_len ctr_common.cc_keysched_len +#define ctr_cb ctr_common.cc_iv +#define ctr_remainder ctr_common.cc_remainder +#define ctr_remainder_len ctr_common.cc_remainder_len +#define ctr_lastp ctr_common.cc_lastp +#define ctr_copy_to ctr_common.cc_copy_to +#define ctr_flags ctr_common.cc_flags + +/* + * + * ccm_mac_len: Stores length of the MAC in CCM mode. + * ccm_mac_buf: Stores the intermediate value for MAC in CCM encrypt. + * In CCM decrypt, stores the input MAC value. + * ccm_data_len: Length of the plaintext for CCM mode encrypt, or + * length of the ciphertext for CCM mode decrypt. + * ccm_processed_data_len: + * Length of processed plaintext in CCM mode encrypt, + * or length of processed ciphertext for CCM mode decrypt. + * ccm_processed_mac_len: + * Length of MAC data accumulated in CCM mode decrypt. + * + * ccm_pt_buf: Only used in CCM mode decrypt. It stores the + * decrypted plaintext to be returned when + * MAC verification succeeds in decrypt_final. + * Memory for this should be allocated in the AES module. + * + */ +typedef struct ccm_ctx { + struct common_ctx ccm_common; + uint32_t ccm_tmp[4]; + size_t ccm_mac_len; + uint64_t ccm_mac_buf[2]; + size_t ccm_data_len; + size_t ccm_processed_data_len; + size_t ccm_processed_mac_len; + uint8_t *ccm_pt_buf; + uint64_t ccm_mac_input_buf[2]; + uint64_t ccm_counter_mask; +} ccm_ctx_t; + +#define ccm_keysched ccm_common.cc_keysched +#define ccm_keysched_len ccm_common.cc_keysched_len +#define ccm_cb ccm_common.cc_iv +#define ccm_remainder ccm_common.cc_remainder +#define ccm_remainder_len ccm_common.cc_remainder_len +#define ccm_lastp ccm_common.cc_lastp +#define ccm_copy_to ccm_common.cc_copy_to +#define ccm_flags ccm_common.cc_flags + +/* + * gcm_tag_len: Length of authentication tag. + * + * gcm_ghash: Stores output from the GHASH function. + * + * gcm_processed_data_len: + * Length of processed plaintext (encrypt) or + * length of processed ciphertext (decrypt). + * + * gcm_pt_buf: Stores the decrypted plaintext returned by + * decrypt_final when the computed authentication + * tag matches the user supplied tag. + * + * gcm_pt_buf_len: Length of the plaintext buffer. + * + * gcm_H: Subkey. + * + * gcm_J0: Pre-counter block generated from the IV. + * + * gcm_len_a_len_c: 64-bit representations of the bit lengths of + * AAD and ciphertext. + * + * gcm_kmflag: Current value of kmflag. Used only for allocating + * the plaintext buffer during decryption. + */ +typedef struct gcm_ctx { + struct common_ctx gcm_common; + size_t gcm_tag_len; + size_t gcm_processed_data_len; + size_t gcm_pt_buf_len; + uint32_t gcm_tmp[4]; + uint64_t gcm_ghash[2]; + uint64_t gcm_H[2]; + uint64_t gcm_J0[2]; + uint64_t gcm_len_a_len_c[2]; + uint8_t *gcm_pt_buf; + int gcm_kmflag; +} gcm_ctx_t; + +#define gcm_keysched gcm_common.cc_keysched +#define gcm_keysched_len gcm_common.cc_keysched_len +#define gcm_cb gcm_common.cc_iv +#define gcm_remainder gcm_common.cc_remainder +#define gcm_remainder_len gcm_common.cc_remainder_len +#define gcm_lastp gcm_common.cc_lastp +#define gcm_copy_to gcm_common.cc_copy_to +#define gcm_flags gcm_common.cc_flags + +#define AES_GMAC_IV_LEN 12 +#define AES_GMAC_TAG_BITS 128 + +typedef struct aes_ctx { + union { + ecb_ctx_t acu_ecb; + cbc_ctx_t acu_cbc; + ctr_ctx_t acu_ctr; + ccm_ctx_t acu_ccm; + gcm_ctx_t acu_gcm; + } acu; +} aes_ctx_t; + +#define ac_flags acu.acu_ecb.ecb_common.cc_flags +#define ac_remainder_len acu.acu_ecb.ecb_common.cc_remainder_len +#define ac_keysched acu.acu_ecb.ecb_common.cc_keysched +#define ac_keysched_len acu.acu_ecb.ecb_common.cc_keysched_len +#define ac_iv acu.acu_ecb.ecb_common.cc_iv +#define ac_lastp acu.acu_ecb.ecb_common.cc_lastp +#define ac_pt_buf acu.acu_ccm.ccm_pt_buf +#define ac_mac_len acu.acu_ccm.ccm_mac_len +#define ac_data_len acu.acu_ccm.ccm_data_len +#define ac_processed_mac_len acu.acu_ccm.ccm_processed_mac_len +#define ac_processed_data_len acu.acu_ccm.ccm_processed_data_len +#define ac_tag_len acu.acu_gcm.gcm_tag_len + +typedef struct blowfish_ctx { + union { + ecb_ctx_t bcu_ecb; + cbc_ctx_t bcu_cbc; + } bcu; +} blowfish_ctx_t; + +#define bc_flags bcu.bcu_ecb.ecb_common.cc_flags +#define bc_remainder_len bcu.bcu_ecb.ecb_common.cc_remainder_len +#define bc_keysched bcu.bcu_ecb.ecb_common.cc_keysched +#define bc_keysched_len bcu.bcu_ecb.ecb_common.cc_keysched_len +#define bc_iv bcu.bcu_ecb.ecb_common.cc_iv +#define bc_lastp bcu.bcu_ecb.ecb_common.cc_lastp + +typedef struct des_ctx { + union { + ecb_ctx_t dcu_ecb; + cbc_ctx_t dcu_cbc; + } dcu; +} des_ctx_t; + +#define dc_flags dcu.dcu_ecb.ecb_common.cc_flags +#define dc_remainder_len dcu.dcu_ecb.ecb_common.cc_remainder_len +#define dc_keysched dcu.dcu_ecb.ecb_common.cc_keysched +#define dc_keysched_len dcu.dcu_ecb.ecb_common.cc_keysched_len +#define dc_iv dcu.dcu_ecb.ecb_common.cc_iv +#define dc_lastp dcu.dcu_ecb.ecb_common.cc_lastp + +extern int ecb_cipher_contiguous_blocks(ecb_ctx_t *, char *, size_t, + crypto_data_t *, size_t, int (*cipher)(const void *, const uint8_t *, + uint8_t *)); + +extern int cbc_encrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t, + crypto_data_t *, size_t, + int (*encrypt)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int cbc_decrypt_contiguous_blocks(cbc_ctx_t *, char *, size_t, + crypto_data_t *, size_t, + int (*decrypt)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int ctr_mode_contiguous_blocks(ctr_ctx_t *, char *, size_t, + crypto_data_t *, size_t, + int (*cipher)(const void *, const uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int ccm_mode_encrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t, + crypto_data_t *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int ccm_mode_decrypt_contiguous_blocks(ccm_ctx_t *, char *, size_t, + crypto_data_t *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t, + crypto_data_t *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *, char *, size_t, + crypto_data_t *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +int ccm_encrypt_final(ccm_ctx_t *, crypto_data_t *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +int gcm_encrypt_final(gcm_ctx_t *, crypto_data_t *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int ccm_decrypt_final(ccm_ctx_t *, crypto_data_t *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int gcm_decrypt_final(gcm_ctx_t *, crypto_data_t *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int ctr_mode_final(ctr_ctx_t *, crypto_data_t *, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)); + +extern int cbc_init_ctx(cbc_ctx_t *, char *, size_t, size_t, + void (*copy_block)(uint8_t *, uint64_t *)); + +extern int ctr_init_ctx(ctr_ctx_t *, ulong_t, uint8_t *, + void (*copy_block)(uint8_t *, uint8_t *)); + +extern int ccm_init_ctx(ccm_ctx_t *, char *, int, boolean_t, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int gcm_init_ctx(gcm_ctx_t *, char *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern int gmac_init_ctx(gcm_ctx_t *, char *, size_t, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), + void (*copy_block)(uint8_t *, uint8_t *), + void (*xor_block)(uint8_t *, uint8_t *)); + +extern void calculate_ccm_mac(ccm_ctx_t *, uint8_t *, + int (*encrypt_block)(const void *, const uint8_t *, uint8_t *)); + +extern void gcm_mul(uint64_t *, uint64_t *, uint64_t *); + +extern void crypto_init_ptrs(crypto_data_t *, void **, offset_t *); +extern void crypto_get_ptrs(crypto_data_t *, void **, offset_t *, + uint8_t **, size_t *, uint8_t **, size_t); + +extern void *ecb_alloc_ctx(int); +extern void *cbc_alloc_ctx(int); +extern void *ctr_alloc_ctx(int); +extern void *ccm_alloc_ctx(int); +extern void *gcm_alloc_ctx(int); +extern void *gmac_alloc_ctx(int); +extern void crypto_free_mode_ctx(void *); +extern void gcm_set_kmflag(gcm_ctx_t *, int); + +#ifdef __cplusplus +} +#endif + +#endif /* _COMMON_CRYPTO_MODES_H */ diff --git a/module/icp/include/sha1/sha1.h b/module/icp/include/sha1/sha1.h new file mode 100644 index 000000000000..b6ae6b8d24e0 --- /dev/null +++ b/module/icp/include/sha1/sha1.h @@ -0,0 +1,61 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_SHA1_H +#define _SYS_SHA1_H + +#include /* for uint_* */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * NOTE: n2rng (Niagara2 RNG driver) accesses the state field of + * SHA1_CTX directly. NEVER change this structure without verifying + * compatiblity with n2rng. The important thing is that the state + * must be in a field declared as uint32_t state[5]. + */ +/* SHA-1 context. */ +typedef struct { + uint32_t state[5]; /* state (ABCDE) */ + uint32_t count[2]; /* number of bits, modulo 2^64 (msb first) */ + union { + uint8_t buf8[64]; /* undigested input */ + uint32_t buf32[16]; /* realigned input */ + } buf_un; +} SHA1_CTX; + +#define SHA1_DIGEST_LENGTH 20 + +void SHA1Init(SHA1_CTX *); +void SHA1Update(SHA1_CTX *, const void *, size_t); +void SHA1Final(void *, SHA1_CTX *); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_SHA1_H */ diff --git a/module/icp/include/sha1/sha1_consts.h b/module/icp/include/sha1/sha1_consts.h new file mode 100644 index 000000000000..848d25ef050f --- /dev/null +++ b/module/icp/include/sha1/sha1_consts.h @@ -0,0 +1,65 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 1998, by Sun Microsystems, Inc. + * All rights reserved. + */ + +#ifndef _SYS_SHA1_CONSTS_H +#define _SYS_SHA1_CONSTS_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * as explained in sha1.c, loading 32-bit constants on a sparc is expensive + * since it involves both a `sethi' and an `or'. thus, we instead use `ld' + * to load the constants from an array called `sha1_consts'. however, on + * intel (and perhaps other processors), it is cheaper to load the constant + * directly. thus, the c code in SHA1Transform() uses the macro SHA1_CONST() + * which either expands to a constant or an array reference, depending on + * the architecture the code is being compiled for. + */ + +#include /* uint32_t */ + +extern const uint32_t sha1_consts[]; + +#if defined(__sparc) +#define SHA1_CONST(x) (sha1_consts[x]) +#else +#define SHA1_CONST(x) (SHA1_CONST_ ## x) +#endif + +/* constants, as provided in FIPS 180-1 */ + +#define SHA1_CONST_0 0x5a827999U +#define SHA1_CONST_1 0x6ed9eba1U +#define SHA1_CONST_2 0x8f1bbcdcU +#define SHA1_CONST_3 0xca62c1d6U + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_SHA1_CONSTS_H */ diff --git a/module/icp/include/sha1/sha1_impl.h b/module/icp/include/sha1/sha1_impl.h new file mode 100644 index 000000000000..1c1f8728f9b5 --- /dev/null +++ b/module/icp/include/sha1/sha1_impl.h @@ -0,0 +1,73 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SHA1_IMPL_H +#define _SHA1_IMPL_H + + +#ifdef __cplusplus +extern "C" { +#endif + +#define SHA1_HASH_SIZE 20 /* SHA_1 digest length in bytes */ +#define SHA1_DIGEST_LENGTH 20 /* SHA1 digest length in bytes */ +#define SHA1_HMAC_BLOCK_SIZE 64 /* SHA1-HMAC block size */ +#define SHA1_HMAC_MIN_KEY_LEN 1 /* SHA1-HMAC min key length in bytes */ +#define SHA1_HMAC_MAX_KEY_LEN INT_MAX /* SHA1-HMAC max key length in bytes */ +#define SHA1_HMAC_INTS_PER_BLOCK (SHA1_HMAC_BLOCK_SIZE/sizeof (uint32_t)) + +/* + * CSPI information (entry points, provider info, etc.) + */ +typedef enum sha1_mech_type { + SHA1_MECH_INFO_TYPE, /* SUN_CKM_SHA1 */ + SHA1_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA1_HMAC */ + SHA1_HMAC_GEN_MECH_INFO_TYPE /* SUN_CKM_SHA1_HMAC_GENERAL */ +} sha1_mech_type_t; + +/* + * Context for SHA1 mechanism. + */ +typedef struct sha1_ctx { + sha1_mech_type_t sc_mech_type; /* type of context */ + SHA1_CTX sc_sha1_ctx; /* SHA1 context */ +} sha1_ctx_t; + +/* + * Context for SHA1-HMAC and SHA1-HMAC-GENERAL mechanisms. + */ +typedef struct sha1_hmac_ctx { + sha1_mech_type_t hc_mech_type; /* type of context */ + uint32_t hc_digest_len; /* digest len in bytes */ + SHA1_CTX hc_icontext; /* inner SHA1 context */ + SHA1_CTX hc_ocontext; /* outer SHA1 context */ +} sha1_hmac_ctx_t; + + +#ifdef __cplusplus +} +#endif + +#endif /* _SHA1_IMPL_H */ diff --git a/module/icp/include/sha2/sha2.h b/module/icp/include/sha2/sha2.h new file mode 100644 index 000000000000..8e53987a7413 --- /dev/null +++ b/module/icp/include/sha2/sha2.h @@ -0,0 +1,116 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ +/* Copyright 2013 Saso Kiselkov. All rights reserved. */ + +#ifndef _SYS_SHA2_H +#define _SYS_SHA2_H + +#include /* for uint_* */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define SHA2_HMAC_MIN_KEY_LEN 1 /* SHA2-HMAC min key length in bytes */ +#define SHA2_HMAC_MAX_KEY_LEN INT_MAX /* SHA2-HMAC max key length in bytes */ + +#define SHA256_DIGEST_LENGTH 32 /* SHA256 digest length in bytes */ + +#define SHA256_HMAC_BLOCK_SIZE 64 /* SHA256-HMAC block size */ + +#define SHA256 0 +#define SHA256_HMAC 1 +#define SHA256_HMAC_GEN 2 + +/* + * SHA2 context. + * The contents of this structure are a private interface between the + * Init/Update/Final calls of the functions defined below. + * Callers must never attempt to read or write any of the fields + * in this structure directly. + */ +typedef struct { + uint32_t algotype; /* Algorithm Type */ + + /* state (ABCDEFGH) */ + union { + uint32_t s32[8]; /* for SHA256 */ + uint64_t s64[8]; /* for SHA384/512 */ + } state; + /* number of bits */ + union { + uint32_t c32[2]; /* for SHA256 , modulo 2^64 */ + uint64_t c64[2]; /* for SHA384/512, modulo 2^128 */ + } count; + union { + uint8_t buf8[128]; /* undigested input */ + uint32_t buf32[32]; /* realigned input */ + uint64_t buf64[16]; /* realigned input */ + } buf_un; +} SHA2_CTX; + +typedef SHA2_CTX SHA256_CTX; +typedef SHA2_CTX SHA384_CTX; +typedef SHA2_CTX SHA512_CTX; + +extern void SHA2Init(uint64_t mech, SHA2_CTX *); + +extern void SHA2Update(SHA2_CTX *, const void *, size_t); + +extern void SHA2Final(void *, SHA2_CTX *); + +extern void SHA256Init(SHA256_CTX *); + +extern void SHA256Update(SHA256_CTX *, const void *, size_t); + +extern void SHA256Final(void *, SHA256_CTX *); + +#ifdef _SHA2_IMPL +/* + * The following types/functions are all private to the implementation + * of the SHA2 functions and must not be used by consumers of the interface + */ + +/* + * List of support mechanisms in this module. + * + * It is important to note that in the module, division or modulus calculations + * are used on the enumerated type to determine which mechanism is being used; + * therefore, changing the order or additional mechanisms should be done + * carefully + */ +typedef enum sha2_mech_type { + SHA256_MECH_INFO_TYPE, /* SUN_CKM_SHA256 */ + SHA256_HMAC_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC */ + SHA256_HMAC_GEN_MECH_INFO_TYPE, /* SUN_CKM_SHA256_HMAC_GENERAL */ +} sha2_mech_type_t; + +#endif /* _SHA2_IMPL */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_SHA2_H */ diff --git a/module/icp/include/sha2/sha2_consts.h b/module/icp/include/sha2/sha2_consts.h new file mode 100644 index 000000000000..3a6645508fe9 --- /dev/null +++ b/module/icp/include/sha2/sha2_consts.h @@ -0,0 +1,219 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_SHA2_CONSTS_H +#define _SYS_SHA2_CONSTS_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Loading 32-bit constants on a sparc is expensive since it involves both + * a `sethi' and an `or'. thus, we instead use `ld' to load the constants + * from an array called `sha2_consts'. however, on intel (and perhaps other + * processors), it is cheaper to load the constant directly. thus, the c + * code in SHA transform functions uses the macro SHA2_CONST() which either + * expands to a constant or an array reference, depending on + * the architecture the code is being compiled for. + * + * SHA512 constants are used for SHA384 + */ + +#include /* uint32_t */ + +extern const uint32_t sha256_consts[]; +extern const uint64_t sha512_consts[]; + +#if defined(__sparc) +#define SHA256_CONST(x) (sha256_consts[x]) +#define SHA512_CONST(x) (sha512_consts[x]) +#else +#define SHA256_CONST(x) (SHA256_CONST_ ## x) +#define SHA512_CONST(x) (SHA512_CONST_ ## x) +#endif + +/* constants, as provided in FIPS 180-2 */ + +#define SHA256_CONST_0 0x428a2f98U +#define SHA256_CONST_1 0x71374491U +#define SHA256_CONST_2 0xb5c0fbcfU +#define SHA256_CONST_3 0xe9b5dba5U +#define SHA256_CONST_4 0x3956c25bU +#define SHA256_CONST_5 0x59f111f1U +#define SHA256_CONST_6 0x923f82a4U +#define SHA256_CONST_7 0xab1c5ed5U + +#define SHA256_CONST_8 0xd807aa98U +#define SHA256_CONST_9 0x12835b01U +#define SHA256_CONST_10 0x243185beU +#define SHA256_CONST_11 0x550c7dc3U +#define SHA256_CONST_12 0x72be5d74U +#define SHA256_CONST_13 0x80deb1feU +#define SHA256_CONST_14 0x9bdc06a7U +#define SHA256_CONST_15 0xc19bf174U + +#define SHA256_CONST_16 0xe49b69c1U +#define SHA256_CONST_17 0xefbe4786U +#define SHA256_CONST_18 0x0fc19dc6U +#define SHA256_CONST_19 0x240ca1ccU +#define SHA256_CONST_20 0x2de92c6fU +#define SHA256_CONST_21 0x4a7484aaU +#define SHA256_CONST_22 0x5cb0a9dcU +#define SHA256_CONST_23 0x76f988daU + +#define SHA256_CONST_24 0x983e5152U +#define SHA256_CONST_25 0xa831c66dU +#define SHA256_CONST_26 0xb00327c8U +#define SHA256_CONST_27 0xbf597fc7U +#define SHA256_CONST_28 0xc6e00bf3U +#define SHA256_CONST_29 0xd5a79147U +#define SHA256_CONST_30 0x06ca6351U +#define SHA256_CONST_31 0x14292967U + +#define SHA256_CONST_32 0x27b70a85U +#define SHA256_CONST_33 0x2e1b2138U +#define SHA256_CONST_34 0x4d2c6dfcU +#define SHA256_CONST_35 0x53380d13U +#define SHA256_CONST_36 0x650a7354U +#define SHA256_CONST_37 0x766a0abbU +#define SHA256_CONST_38 0x81c2c92eU +#define SHA256_CONST_39 0x92722c85U + +#define SHA256_CONST_40 0xa2bfe8a1U +#define SHA256_CONST_41 0xa81a664bU +#define SHA256_CONST_42 0xc24b8b70U +#define SHA256_CONST_43 0xc76c51a3U +#define SHA256_CONST_44 0xd192e819U +#define SHA256_CONST_45 0xd6990624U +#define SHA256_CONST_46 0xf40e3585U +#define SHA256_CONST_47 0x106aa070U + +#define SHA256_CONST_48 0x19a4c116U +#define SHA256_CONST_49 0x1e376c08U +#define SHA256_CONST_50 0x2748774cU +#define SHA256_CONST_51 0x34b0bcb5U +#define SHA256_CONST_52 0x391c0cb3U +#define SHA256_CONST_53 0x4ed8aa4aU +#define SHA256_CONST_54 0x5b9cca4fU +#define SHA256_CONST_55 0x682e6ff3U + +#define SHA256_CONST_56 0x748f82eeU +#define SHA256_CONST_57 0x78a5636fU +#define SHA256_CONST_58 0x84c87814U +#define SHA256_CONST_59 0x8cc70208U +#define SHA256_CONST_60 0x90befffaU +#define SHA256_CONST_61 0xa4506cebU +#define SHA256_CONST_62 0xbef9a3f7U +#define SHA256_CONST_63 0xc67178f2U + +#define SHA512_CONST_0 0x428a2f98d728ae22ULL +#define SHA512_CONST_1 0x7137449123ef65cdULL +#define SHA512_CONST_2 0xb5c0fbcfec4d3b2fULL +#define SHA512_CONST_3 0xe9b5dba58189dbbcULL +#define SHA512_CONST_4 0x3956c25bf348b538ULL +#define SHA512_CONST_5 0x59f111f1b605d019ULL +#define SHA512_CONST_6 0x923f82a4af194f9bULL +#define SHA512_CONST_7 0xab1c5ed5da6d8118ULL +#define SHA512_CONST_8 0xd807aa98a3030242ULL +#define SHA512_CONST_9 0x12835b0145706fbeULL +#define SHA512_CONST_10 0x243185be4ee4b28cULL +#define SHA512_CONST_11 0x550c7dc3d5ffb4e2ULL +#define SHA512_CONST_12 0x72be5d74f27b896fULL +#define SHA512_CONST_13 0x80deb1fe3b1696b1ULL +#define SHA512_CONST_14 0x9bdc06a725c71235ULL +#define SHA512_CONST_15 0xc19bf174cf692694ULL +#define SHA512_CONST_16 0xe49b69c19ef14ad2ULL +#define SHA512_CONST_17 0xefbe4786384f25e3ULL +#define SHA512_CONST_18 0x0fc19dc68b8cd5b5ULL +#define SHA512_CONST_19 0x240ca1cc77ac9c65ULL +#define SHA512_CONST_20 0x2de92c6f592b0275ULL +#define SHA512_CONST_21 0x4a7484aa6ea6e483ULL +#define SHA512_CONST_22 0x5cb0a9dcbd41fbd4ULL +#define SHA512_CONST_23 0x76f988da831153b5ULL +#define SHA512_CONST_24 0x983e5152ee66dfabULL +#define SHA512_CONST_25 0xa831c66d2db43210ULL +#define SHA512_CONST_26 0xb00327c898fb213fULL +#define SHA512_CONST_27 0xbf597fc7beef0ee4ULL +#define SHA512_CONST_28 0xc6e00bf33da88fc2ULL +#define SHA512_CONST_29 0xd5a79147930aa725ULL +#define SHA512_CONST_30 0x06ca6351e003826fULL +#define SHA512_CONST_31 0x142929670a0e6e70ULL +#define SHA512_CONST_32 0x27b70a8546d22ffcULL +#define SHA512_CONST_33 0x2e1b21385c26c926ULL +#define SHA512_CONST_34 0x4d2c6dfc5ac42aedULL +#define SHA512_CONST_35 0x53380d139d95b3dfULL +#define SHA512_CONST_36 0x650a73548baf63deULL +#define SHA512_CONST_37 0x766a0abb3c77b2a8ULL +#define SHA512_CONST_38 0x81c2c92e47edaee6ULL +#define SHA512_CONST_39 0x92722c851482353bULL +#define SHA512_CONST_40 0xa2bfe8a14cf10364ULL +#define SHA512_CONST_41 0xa81a664bbc423001ULL +#define SHA512_CONST_42 0xc24b8b70d0f89791ULL +#define SHA512_CONST_43 0xc76c51a30654be30ULL +#define SHA512_CONST_44 0xd192e819d6ef5218ULL +#define SHA512_CONST_45 0xd69906245565a910ULL +#define SHA512_CONST_46 0xf40e35855771202aULL +#define SHA512_CONST_47 0x106aa07032bbd1b8ULL +#define SHA512_CONST_48 0x19a4c116b8d2d0c8ULL +#define SHA512_CONST_49 0x1e376c085141ab53ULL +#define SHA512_CONST_50 0x2748774cdf8eeb99ULL +#define SHA512_CONST_51 0x34b0bcb5e19b48a8ULL +#define SHA512_CONST_52 0x391c0cb3c5c95a63ULL +#define SHA512_CONST_53 0x4ed8aa4ae3418acbULL +#define SHA512_CONST_54 0x5b9cca4f7763e373ULL +#define SHA512_CONST_55 0x682e6ff3d6b2b8a3ULL +#define SHA512_CONST_56 0x748f82ee5defb2fcULL +#define SHA512_CONST_57 0x78a5636f43172f60ULL +#define SHA512_CONST_58 0x84c87814a1f0ab72ULL +#define SHA512_CONST_59 0x8cc702081a6439ecULL +#define SHA512_CONST_60 0x90befffa23631e28ULL +#define SHA512_CONST_61 0xa4506cebde82bde9ULL +#define SHA512_CONST_62 0xbef9a3f7b2c67915ULL +#define SHA512_CONST_63 0xc67178f2e372532bULL +#define SHA512_CONST_64 0xca273eceea26619cULL +#define SHA512_CONST_65 0xd186b8c721c0c207ULL +#define SHA512_CONST_66 0xeada7dd6cde0eb1eULL +#define SHA512_CONST_67 0xf57d4f7fee6ed178ULL +#define SHA512_CONST_68 0x06f067aa72176fbaULL +#define SHA512_CONST_69 0x0a637dc5a2c898a6ULL +#define SHA512_CONST_70 0x113f9804bef90daeULL +#define SHA512_CONST_71 0x1b710b35131c471bULL +#define SHA512_CONST_72 0x28db77f523047d84ULL +#define SHA512_CONST_73 0x32caab7b40c72493ULL +#define SHA512_CONST_74 0x3c9ebe0a15c9bebcULL +#define SHA512_CONST_75 0x431d67c49c100d4cULL +#define SHA512_CONST_76 0x4cc5d4becb3e42b6ULL +#define SHA512_CONST_77 0x597f299cfc657e2aULL +#define SHA512_CONST_78 0x5fcb6fab3ad6faecULL +#define SHA512_CONST_79 0x6c44198c4a475817ULL + + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_SHA2_CONSTS_H */ diff --git a/module/icp/include/sha2/sha2_impl.h b/module/icp/include/sha2/sha2_impl.h new file mode 100644 index 000000000000..bb42c3cd4bc3 --- /dev/null +++ b/module/icp/include/sha2/sha2_impl.h @@ -0,0 +1,62 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SHA2_IMPL_H +#define _SHA2_IMPL_H + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { + SHA1_TYPE, + SHA256_TYPE, + SHA384_TYPE, + SHA512_TYPE +} sha2_mech_t; + +/* + * Context for SHA2 mechanism. + */ +typedef struct sha2_ctx { + sha2_mech_type_t sc_mech_type; /* type of context */ + SHA2_CTX sc_sha2_ctx; /* SHA2 context */ +} sha2_ctx_t; + +/* + * Context for SHA2 HMAC and HMAC GENERAL mechanisms. + */ +typedef struct sha2_hmac_ctx { + sha2_mech_type_t hc_mech_type; /* type of context */ + uint32_t hc_digest_len; /* digest len in bytes */ + SHA2_CTX hc_icontext; /* inner SHA2 context */ + SHA2_CTX hc_ocontext; /* outer SHA2 context */ +} sha2_hmac_ctx_t; + +#ifdef __cplusplus +} +#endif + +#endif /* _SHA2_IMPL_H */ diff --git a/lib/libspl/include/priv.h b/module/icp/include/sys/asm_linkage.h similarity index 68% rename from lib/libspl/include/priv.h rename to module/icp/include/sys/asm_linkage.h index 15b76a4006ff..3805978577e6 100644 --- a/lib/libspl/include/priv.h +++ b/module/icp/include/sys/asm_linkage.h @@ -20,27 +20,17 @@ * CDDL HEADER END */ /* - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. */ -#ifndef _LIBSPL_PRIV_H -#define _LIBSPL_PRIV_H +#ifndef _SYS_ASM_LINKAGE_H +#define _SYS_ASM_LINKAGE_H -#include +#if defined(__i386) || defined(__amd64) -/* Couldn't find this definition in OpenGrok */ -#define PRIV_SYS_CONFIG "sys_config" - -/* - * priv_op_t indicates a privilege operation type - */ -typedef enum priv_op { - PRIV_ON, - PRIV_OFF, - PRIV_SET -} priv_op_t; - -static inline boolean_t priv_ineffect(const char *priv) { return B_TRUE; } +#include /* XX64 x86/sys/asm_linkage.h */ #endif + +#endif /* _SYS_ASM_LINKAGE_H */ diff --git a/module/icp/include/sys/bitmap.h b/module/icp/include/sys/bitmap.h new file mode 100644 index 000000000000..b1f6823e61d4 --- /dev/null +++ b/module/icp/include/sys/bitmap.h @@ -0,0 +1,183 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ +/* All Rights Reserved */ + + +#ifndef _SYS_BITMAP_H +#define _SYS_BITMAP_H + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(__GNUC__) && defined(_ASM_INLINES) && \ + (defined(__i386) || defined(__amd64)) +#include +#endif + +/* + * Operations on bitmaps of arbitrary size + * A bitmap is a vector of 1 or more ulong_t's. + * The user of the package is responsible for range checks and keeping + * track of sizes. + */ + +#ifdef _LP64 +#define BT_ULSHIFT 6 /* log base 2 of BT_NBIPUL, to extract word index */ +#define BT_ULSHIFT32 5 /* log base 2 of BT_NBIPUL, to extract word index */ +#else +#define BT_ULSHIFT 5 /* log base 2 of BT_NBIPUL, to extract word index */ +#endif + +#define BT_NBIPUL (1 << BT_ULSHIFT) /* n bits per ulong_t */ +#define BT_ULMASK (BT_NBIPUL - 1) /* to extract bit index */ + +#ifdef _LP64 +#define BT_NBIPUL32 (1 << BT_ULSHIFT32) /* n bits per ulong_t */ +#define BT_ULMASK32 (BT_NBIPUL32 - 1) /* to extract bit index */ +#define BT_ULMAXMASK 0xffffffffffffffff /* used by bt_getlowbit */ +#else +#define BT_ULMAXMASK 0xffffffff +#endif + +/* + * bitmap is a ulong_t *, bitindex an index_t + * + * The macros BT_WIM and BT_BIW internal; there is no need + * for users of this package to use them. + */ + +/* + * word in map + */ +#define BT_WIM(bitmap, bitindex) \ + ((bitmap)[(bitindex) >> BT_ULSHIFT]) +/* + * bit in word + */ +#define BT_BIW(bitindex) \ + (1UL << ((bitindex) & BT_ULMASK)) + +#ifdef _LP64 +#define BT_WIM32(bitmap, bitindex) \ + ((bitmap)[(bitindex) >> BT_ULSHIFT32]) + +#define BT_BIW32(bitindex) \ + (1UL << ((bitindex) & BT_ULMASK32)) +#endif + +/* + * These are public macros + * + * BT_BITOUL == n bits to n ulong_t's + */ +#define BT_BITOUL(nbits) \ + (((nbits) + BT_NBIPUL - 1l) / BT_NBIPUL) +#define BT_SIZEOFMAP(nbits) \ + (BT_BITOUL(nbits) * sizeof (ulong_t)) +#define BT_TEST(bitmap, bitindex) \ + ((BT_WIM((bitmap), (bitindex)) & BT_BIW(bitindex)) ? 1 : 0) +#define BT_SET(bitmap, bitindex) \ + { BT_WIM((bitmap), (bitindex)) |= BT_BIW(bitindex); } +#define BT_CLEAR(bitmap, bitindex) \ + { BT_WIM((bitmap), (bitindex)) &= ~BT_BIW(bitindex); } + +#ifdef _LP64 +#define BT_BITOUL32(nbits) \ + (((nbits) + BT_NBIPUL32 - 1l) / BT_NBIPUL32) +#define BT_SIZEOFMAP32(nbits) \ + (BT_BITOUL32(nbits) * sizeof (uint_t)) +#define BT_TEST32(bitmap, bitindex) \ + ((BT_WIM32((bitmap), (bitindex)) & BT_BIW32(bitindex)) ? 1 : 0) +#define BT_SET32(bitmap, bitindex) \ + { BT_WIM32((bitmap), (bitindex)) |= BT_BIW32(bitindex); } +#define BT_CLEAR32(bitmap, bitindex) \ + { BT_WIM32((bitmap), (bitindex)) &= ~BT_BIW32(bitindex); } +#endif /* _LP64 */ + + +/* + * BIT_ONLYONESET is a private macro not designed for bitmaps of + * arbitrary size. u must be an unsigned integer/long. It returns + * true if one and only one bit is set in u. + */ +#define BIT_ONLYONESET(u) \ + ((((u) == 0) ? 0 : ((u) & ((u) - 1)) == 0)) + +#ifndef _ASM + +/* + * return next available bit index from map with specified number of bits + */ +extern index_t bt_availbit(ulong_t *bitmap, size_t nbits); +/* + * find the highest order bit that is on, and is within or below + * the word specified by wx + */ +extern int bt_gethighbit(ulong_t *mapp, int wx); +extern int bt_range(ulong_t *bitmap, size_t *pos1, size_t *pos2, + size_t end_pos); +extern int bt_getlowbit(ulong_t *bitmap, size_t start, size_t stop); +extern void bt_copy(ulong_t *, ulong_t *, ulong_t); + +/* + * find the parity + */ +extern int odd_parity(ulong_t); + +/* + * Atomically set/clear bits + * Atomic exclusive operations will set "result" to "-1" + * if the bit is already set/cleared. "result" will be set + * to 0 otherwise. + */ +#define BT_ATOMIC_SET(bitmap, bitindex) \ + { atomic_or_long(&(BT_WIM(bitmap, bitindex)), BT_BIW(bitindex)); } +#define BT_ATOMIC_CLEAR(bitmap, bitindex) \ + { atomic_and_long(&(BT_WIM(bitmap, bitindex)), ~BT_BIW(bitindex)); } + +#define BT_ATOMIC_SET_EXCL(bitmap, bitindex, result) \ + { result = atomic_set_long_excl(&(BT_WIM(bitmap, bitindex)), \ + (bitindex) % BT_NBIPUL); } +#define BT_ATOMIC_CLEAR_EXCL(bitmap, bitindex, result) \ + { result = atomic_clear_long_excl(&(BT_WIM(bitmap, bitindex)), \ + (bitindex) % BT_NBIPUL); } + +/* + * Extracts bits between index h (high, inclusive) and l (low, exclusive) from + * u, which must be an unsigned integer. + */ +#define BITX(u, h, l) (((u) >> (l)) & ((1LU << ((h) - (l) + 1LU)) - 1LU)) + +#endif /* _ASM */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_BITMAP_H */ diff --git a/module/icp/include/sys/crypto/elfsign.h b/module/icp/include/sys/crypto/elfsign.h new file mode 100644 index 000000000000..5432f0c8d607 --- /dev/null +++ b/module/icp/include/sys/crypto/elfsign.h @@ -0,0 +1,137 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CRYPTO_ELFSIGN_H +#define _SYS_CRYPTO_ELFSIGN_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Consolidation Private Interface for elfsign/libpkcs11/kcfd + */ + +#include + +/* + * Project Private structures and types used for communication between kcfd + * and KCF over the door. + */ + +typedef enum ELFsign_status_e { + ELFSIGN_UNKNOWN, + ELFSIGN_SUCCESS, + ELFSIGN_FAILED, + ELFSIGN_NOTSIGNED, + ELFSIGN_INVALID_CERTPATH, + ELFSIGN_INVALID_ELFOBJ, + ELFSIGN_RESTRICTED +} ELFsign_status_t; + +#define KCF_KCFD_VERSION1 1 +#define SIG_MAX_LENGTH 1024 + +#define ELF_SIGNATURE_SECTION ".SUNW_signature" + +typedef struct kcf_door_arg_s { + short da_version; + boolean_t da_iskernel; + + union { + char filename[MAXPATHLEN]; /* For request */ + + struct kcf_door_result_s { /* For response */ + ELFsign_status_t status; + uint32_t siglen; + uchar_t signature[1]; + } result; + } da_u; +} kcf_door_arg_t; + +typedef uint32_t filesig_vers_t; + +/* + * File Signature Structure + * Applicable to ELF and other file formats + */ +struct filesignatures { + uint32_t filesig_cnt; /* count of signatures */ + uint32_t filesig_pad; /* unused */ + union { + char filesig_data[1]; + struct filesig { /* one of these for each signature */ + uint32_t filesig_size; + filesig_vers_t filesig_version; + union { + struct filesig_version1 { + uint32_t filesig_v1_dnsize; + uint32_t filesig_v1_sigsize; + uint32_t filesig_v1_oidsize; + char filesig_v1_data[1]; + } filesig_v1; + struct filesig_version3 { + uint64_t filesig_v3_time; + uint32_t filesig_v3_dnsize; + uint32_t filesig_v3_sigsize; + uint32_t filesig_v3_oidsize; + char filesig_v3_data[1]; + } filesig_v3; + } _u2; + } filesig_sig; + uint64_t filesig_align; + } _u1; +}; +#define filesig_sig _u1.filesig_sig + +#define filesig_v1_dnsize _u2.filesig_v1.filesig_v1_dnsize +#define filesig_v1_sigsize _u2.filesig_v1.filesig_v1_sigsize +#define filesig_v1_oidsize _u2.filesig_v1.filesig_v1_oidsize +#define filesig_v1_data _u2.filesig_v1.filesig_v1_data + +#define filesig_v3_time _u2.filesig_v3.filesig_v3_time +#define filesig_v3_dnsize _u2.filesig_v3.filesig_v3_dnsize +#define filesig_v3_sigsize _u2.filesig_v3.filesig_v3_sigsize +#define filesig_v3_oidsize _u2.filesig_v3.filesig_v3_oidsize +#define filesig_v3_data _u2.filesig_v3.filesig_v3_data + +#define filesig_ALIGN(s) (((s) + sizeof (uint64_t) - 1) & \ + (-sizeof (uint64_t))) +#define filesig_next(ptr) (struct filesig *)((void *)((char *)(ptr) + \ + filesig_ALIGN((ptr)->filesig_size))) + +#define FILESIG_UNKNOWN 0 /* unrecognized version */ +#define FILESIG_VERSION1 1 /* version1, all but sig section */ +#define FILESIG_VERSION2 2 /* version1 format, SHF_ALLOC only */ +#define FILESIG_VERSION3 3 /* version3, all but sig section */ +#define FILESIG_VERSION4 4 /* version3 format, SHF_ALLOC only */ + +#define _PATH_KCFD_DOOR "/etc/svc/volatile/kcfd_door" + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRYPTO_ELFSIGN_H */ diff --git a/module/icp/include/sys/crypto/impl.h b/module/icp/include/sys/crypto/impl.h new file mode 100644 index 000000000000..258cb5fedcd0 --- /dev/null +++ b/module/icp/include/sys/crypto/impl.h @@ -0,0 +1,1363 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CRYPTO_IMPL_H +#define _SYS_CRYPTO_IMPL_H + +/* + * Kernel Cryptographic Framework private implementation definitions. + */ + +#include +#include +#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define KCF_MODULE "kcf" + +/* + * Prefixes convention: structures internal to the kernel cryptographic + * framework start with 'kcf_'. Exposed structure start with 'crypto_'. + */ + +/* Provider stats. Not protected. */ +typedef struct kcf_prov_stats { + kstat_named_t ps_ops_total; + kstat_named_t ps_ops_passed; + kstat_named_t ps_ops_failed; + kstat_named_t ps_ops_busy_rval; +} kcf_prov_stats_t; + +/* Various kcf stats. Not protected. */ +typedef struct kcf_stats { + kstat_named_t ks_thrs_in_pool; + kstat_named_t ks_idle_thrs; + kstat_named_t ks_minthrs; + kstat_named_t ks_maxthrs; + kstat_named_t ks_swq_njobs; + kstat_named_t ks_swq_maxjobs; + kstat_named_t ks_taskq_threads; + kstat_named_t ks_taskq_minalloc; + kstat_named_t ks_taskq_maxalloc; +} kcf_stats_t; + +/* + * Keep all the information needed by the scheduler from + * this provider. + */ +typedef struct kcf_sched_info { + /* The number of operations dispatched. */ + uint64_t ks_ndispatches; + + /* The number of operations that failed. */ + uint64_t ks_nfails; + + /* The number of operations that returned CRYPTO_BUSY. */ + uint64_t ks_nbusy_rval; + + /* taskq used to dispatch crypto requests */ + taskq_t *ks_taskq; +} kcf_sched_info_t; + +/* + * pd_irefcnt approximates the number of inflight requests to the + * provider. Though we increment this counter during registration for + * other purposes, that base value is mostly same across all providers. + * So, it is a good measure of the load on a provider when it is not + * in a busy state. Once a provider notifies it is busy, requests + * backup in the taskq. So, we use tq_nalloc in that case which gives + * the number of task entries in the task queue. Note that we do not + * acquire any locks here as it is not critical to get the exact number + * and the lock contention may be too costly for this code path. + */ +#define KCF_PROV_LOAD(pd) ((pd)->pd_state != KCF_PROV_BUSY ? \ + (pd)->pd_irefcnt : (pd)->pd_sched_info.ks_taskq->tq_nalloc) + +#define KCF_PROV_INCRSTATS(pd, error) { \ + (pd)->pd_sched_info.ks_ndispatches++; \ + if (error == CRYPTO_BUSY) \ + (pd)->pd_sched_info.ks_nbusy_rval++; \ + else if (error != CRYPTO_SUCCESS && error != CRYPTO_QUEUED) \ + (pd)->pd_sched_info.ks_nfails++; \ +} + + +/* + * The following two macros should be + * #define KCF_OPS_CLASSSIZE (KCF_LAST_OPSCLASS - KCF_FIRST_OPSCLASS + 2) + * #define KCF_MAXMECHTAB KCF_MAXCIPHER + * + * However, doing that would involve reorganizing the header file a bit. + * When impl.h is broken up (bug# 4703218), this will be done. For now, + * we hardcode these values. + */ +#define KCF_OPS_CLASSSIZE 8 +#define KCF_MAXMECHTAB 32 + +/* + * Valid values for the state of a provider. The order of + * the elements is important. + * + * Routines which get a provider or the list of providers + * should pick only those that are either in KCF_PROV_READY state + * or in KCF_PROV_BUSY state. + */ +typedef enum { + KCF_PROV_ALLOCATED = 1, + KCF_PROV_UNVERIFIED, + KCF_PROV_VERIFICATION_FAILED, + /* + * state < KCF_PROV_READY means the provider can not + * be used at all. + */ + KCF_PROV_READY, + KCF_PROV_BUSY, + /* + * state > KCF_PROV_BUSY means the provider can not + * be used for new requests. + */ + KCF_PROV_FAILED, + /* + * Threads setting the following two states should do so only + * if the current state < KCF_PROV_DISABLED. + */ + KCF_PROV_DISABLED, + KCF_PROV_REMOVED, + KCF_PROV_FREED +} kcf_prov_state_t; + +#define KCF_IS_PROV_UNVERIFIED(pd) ((pd)->pd_state == KCF_PROV_UNVERIFIED) +#define KCF_IS_PROV_USABLE(pd) ((pd)->pd_state == KCF_PROV_READY || \ + (pd)->pd_state == KCF_PROV_BUSY) +#define KCF_IS_PROV_REMOVED(pd) ((pd)->pd_state >= KCF_PROV_REMOVED) + +/* Internal flags valid for pd_flags field */ +#define KCF_PROV_RESTRICTED 0x40000000 +#define KCF_LPROV_MEMBER 0x80000000 /* is member of a logical provider */ + +/* + * A provider descriptor structure. There is one such structure per + * provider. It is allocated and initialized at registration time and + * freed when the provider unregisters. + * + * pd_prov_type: Provider type, hardware or software + * pd_sid: Session ID of the provider used by kernel clients. + * This is valid only for session-oriented providers. + * pd_refcnt: Reference counter to this provider descriptor + * pd_irefcnt: References held by the framework internal structs + * pd_lock: lock protects pd_state and pd_provider_list + * pd_state: State value of the provider + * pd_provider_list: Used to cross-reference logical providers and their + * members. Not used for software providers. + * pd_resume_cv: cv to wait for state to change from KCF_PROV_BUSY + * pd_prov_handle: Provider handle specified by provider + * pd_ops_vector: The ops vector specified by Provider + * pd_mech_indx: Lookup table which maps a core framework mechanism + * number to an index in pd_mechanisms array + * pd_mechanisms: Array of mechanisms supported by the provider, specified + * by the provider during registration + * pd_sched_info: Scheduling information associated with the provider + * pd_mech_list_count: The number of entries in pi_mechanisms, specified + * by the provider during registration + * pd_name: Device name or module name + * pd_instance: Device instance + * pd_module_id: Module ID returned by modload + * pd_mctlp: Pointer to modctl structure for this provider + * pd_remove_cv: cv to wait on while the provider queue drains + * pd_description: Provider description string + * pd_flags bitwise OR of pi_flags from crypto_provider_info_t + * and other internal flags defined above. + * pd_hash_limit Maximum data size that hash mechanisms of this provider + * can support. + * pd_kcf_prov_handle: KCF-private handle assigned by KCF + * pd_prov_id: Identification # assigned by KCF to provider + * pd_kstat: kstat associated with the provider + * pd_ks_data: kstat data + */ +typedef struct kcf_provider_desc { + crypto_provider_type_t pd_prov_type; + crypto_session_id_t pd_sid; + uint_t pd_refcnt; + uint_t pd_irefcnt; + kmutex_t pd_lock; + kcf_prov_state_t pd_state; + struct kcf_provider_list *pd_provider_list; + kcondvar_t pd_resume_cv; + crypto_provider_handle_t pd_prov_handle; + crypto_ops_t *pd_ops_vector; + ushort_t pd_mech_indx[KCF_OPS_CLASSSIZE]\ + [KCF_MAXMECHTAB]; + crypto_mech_info_t *pd_mechanisms; + kcf_sched_info_t pd_sched_info; + uint_t pd_mech_list_count; + // char *pd_name; + // uint_t pd_instance; + // int pd_module_id; + // struct modctl *pd_mctlp; + kcondvar_t pd_remove_cv; + char *pd_description; + uint_t pd_flags; + uint_t pd_hash_limit; + crypto_kcf_provider_handle_t pd_kcf_prov_handle; + crypto_provider_id_t pd_prov_id; + kstat_t *pd_kstat; + kcf_prov_stats_t pd_ks_data; +} kcf_provider_desc_t; + +/* useful for making a list of providers */ +typedef struct kcf_provider_list { + struct kcf_provider_list *pl_next; + struct kcf_provider_desc *pl_provider; +} kcf_provider_list_t; + +/* atomic operations in linux implictly form a memory barrier */ +#define membar_exit() + +/* + * If a component has a reference to a kcf_provider_desc_t, + * it REFHOLD()s. A new provider descriptor which is referenced only + * by the providers table has a reference counter of one. + */ +#define KCF_PROV_REFHOLD(desc) { \ + atomic_add_32(&(desc)->pd_refcnt, 1); \ + ASSERT((desc)->pd_refcnt != 0); \ +} + +#define KCF_PROV_IREFHOLD(desc) { \ + atomic_add_32(&(desc)->pd_irefcnt, 1); \ + ASSERT((desc)->pd_irefcnt != 0); \ +} + +#define KCF_PROV_IREFRELE(desc) { \ + ASSERT((desc)->pd_irefcnt != 0); \ + membar_exit(); \ + if (atomic_add_32_nv(&(desc)->pd_irefcnt, -1) == 0) { \ + cv_broadcast(&(desc)->pd_remove_cv); \ + } \ +} + +#define KCF_PROV_REFHELD(desc) ((desc)->pd_refcnt >= 1) + +#define KCF_PROV_REFRELE(desc) { \ + ASSERT((desc)->pd_refcnt != 0); \ + membar_exit(); \ + if (atomic_add_32_nv(&(desc)->pd_refcnt, -1) == 0) { \ + kcf_provider_zero_refcnt((desc)); \ + } \ +} + + +/* list of crypto_mech_info_t valid as the second mech in a dual operation */ + +typedef struct crypto_mech_info_list { + struct crypto_mech_info_list *ml_next; + crypto_mech_type_t ml_kcf_mechid; /* KCF's id */ + crypto_mech_info_t ml_mech_info; +} crypto_mech_info_list_t; + +/* + * An element in a mechanism provider descriptors chain. + * The kcf_prov_mech_desc_t is duplicated in every chain the provider belongs + * to. This is a small tradeoff memory vs mutex spinning time to access the + * common provider field. + */ + +typedef struct kcf_prov_mech_desc { + struct kcf_mech_entry *pm_me; /* Back to the head */ + struct kcf_prov_mech_desc *pm_next; /* Next in the chain */ + crypto_mech_info_t pm_mech_info; /* Provider mech info */ + crypto_mech_info_list_t *pm_mi_list; /* list for duals */ + kcf_provider_desc_t *pm_prov_desc; /* Common desc. */ +} kcf_prov_mech_desc_t; + +/* and the notation shortcuts ... */ +#define pm_provider_type pm_prov_desc.pd_provider_type +#define pm_provider_handle pm_prov_desc.pd_provider_handle +#define pm_ops_vector pm_prov_desc.pd_ops_vector + +/* + * A mechanism entry in an xxx_mech_tab[]. me_pad was deemed + * to be unnecessary and removed. + */ +typedef struct kcf_mech_entry { + crypto_mech_name_t me_name; /* mechanism name */ + crypto_mech_type_t me_mechid; /* Internal id for mechanism */ + kmutex_t me_mutex; /* access protection */ + kcf_prov_mech_desc_t *me_hw_prov_chain; /* list of HW providers */ + kcf_prov_mech_desc_t *me_sw_prov; /* SW provider */ + /* + * Number of HW providers in the chain. There is only one + * SW provider. So, we need only a count of HW providers. + */ + int me_num_hwprov; + /* + * When a SW provider is present, this is the generation number that + * ensures no objects from old SW providers are used in the new one + */ + uint32_t me_gen_swprov; + /* + * threshold for using hardware providers for this mech + */ + size_t me_threshold; +} kcf_mech_entry_t; + +/* + * A policy descriptor structure. It is allocated and initialized + * when administrative ioctls load disabled mechanisms. + * + * pd_prov_type: Provider type, hardware or software + * pd_name: Device name or module name. + * pd_instance: Device instance. + * pd_refcnt: Reference counter for this policy descriptor + * pd_mutex: Protects array and count of disabled mechanisms. + * pd_disabled_count: Count of disabled mechanisms. + * pd_disabled_mechs: Array of disabled mechanisms. + */ +typedef struct kcf_policy_desc { + crypto_provider_type_t pd_prov_type; + char *pd_name; + uint_t pd_instance; + uint_t pd_refcnt; + kmutex_t pd_mutex; + uint_t pd_disabled_count; + crypto_mech_name_t *pd_disabled_mechs; +} kcf_policy_desc_t; + +/* + * If a component has a reference to a kcf_policy_desc_t, + * it REFHOLD()s. A new policy descriptor which is referenced only + * by the policy table has a reference count of one. + */ +#define KCF_POLICY_REFHOLD(desc) { \ + atomic_add_32(&(desc)->pd_refcnt, 1); \ + ASSERT((desc)->pd_refcnt != 0); \ +} + +/* + * Releases a reference to a policy descriptor. When the last + * reference is released, the descriptor is freed. + */ +#define KCF_POLICY_REFRELE(desc) { \ + ASSERT((desc)->pd_refcnt != 0); \ + membar_exit(); \ + if (atomic_add_32_nv(&(desc)->pd_refcnt, -1) == 0) \ + kcf_policy_free_desc(desc); \ +} + +/* + * This entry stores the name of a software module and its + * mechanisms. The mechanisms are 'hints' that are used to + * trigger loading of the module. + */ +typedef struct kcf_soft_conf_entry { + struct kcf_soft_conf_entry *ce_next; + char *ce_name; + crypto_mech_name_t *ce_mechs; + uint_t ce_count; +} kcf_soft_conf_entry_t; + +extern kmutex_t soft_config_mutex; +extern kcf_soft_conf_entry_t *soft_config_list; + +/* + * Global tables. The sizes are from the predefined PKCS#11 v2.20 mechanisms, + * with a margin of few extra empty entry points + */ + +#define KCF_MAXDIGEST 16 /* Digests */ +#define KCF_MAXCIPHER 64 /* Ciphers */ +#define KCF_MAXMAC 40 /* Message authentication codes */ +#define KCF_MAXSIGN 24 /* Sign/Verify */ +#define KCF_MAXKEYOPS 116 /* Key generation and derivation */ +#define KCF_MAXMISC 16 /* Others ... */ + +#define KCF_MAXMECHS KCF_MAXDIGEST + KCF_MAXCIPHER + KCF_MAXMAC + \ + KCF_MAXSIGN + KCF_MAXKEYOPS + \ + KCF_MAXMISC + +extern kcf_mech_entry_t kcf_digest_mechs_tab[]; +extern kcf_mech_entry_t kcf_cipher_mechs_tab[]; +extern kcf_mech_entry_t kcf_mac_mechs_tab[]; +extern kcf_mech_entry_t kcf_sign_mechs_tab[]; +extern kcf_mech_entry_t kcf_keyops_mechs_tab[]; +extern kcf_mech_entry_t kcf_misc_mechs_tab[]; + +extern kmutex_t kcf_mech_tabs_lock; + +typedef enum { + KCF_DIGEST_CLASS = 1, + KCF_CIPHER_CLASS, + KCF_MAC_CLASS, + KCF_SIGN_CLASS, + KCF_KEYOPS_CLASS, + KCF_MISC_CLASS +} kcf_ops_class_t; + +#define KCF_FIRST_OPSCLASS KCF_DIGEST_CLASS +#define KCF_LAST_OPSCLASS KCF_MISC_CLASS + +/* The table of all the kcf_xxx_mech_tab[]s, indexed by kcf_ops_class */ + +typedef struct kcf_mech_entry_tab { + int met_size; /* Size of the met_tab[] */ + kcf_mech_entry_t *met_tab; /* the table */ +} kcf_mech_entry_tab_t; + +extern kcf_mech_entry_tab_t kcf_mech_tabs_tab[]; + +#define KCF_MECHID(class, index) \ + (((crypto_mech_type_t)(class) << 32) | (crypto_mech_type_t)(index)) + +#define KCF_MECH2CLASS(mech_type) ((kcf_ops_class_t)((mech_type) >> 32)) + +#define KCF_MECH2INDEX(mech_type) ((int)(mech_type)) + +#define KCF_TO_PROV_MECH_INDX(pd, mech_type) \ + ((pd)->pd_mech_indx[KCF_MECH2CLASS(mech_type)] \ + [KCF_MECH2INDEX(mech_type)]) + +#define KCF_TO_PROV_MECHINFO(pd, mech_type) \ + ((pd)->pd_mechanisms[KCF_TO_PROV_MECH_INDX(pd, mech_type)]) + +#define KCF_TO_PROV_MECHNUM(pd, mech_type) \ + (KCF_TO_PROV_MECHINFO(pd, mech_type).cm_mech_number) + +#define KCF_CAN_SHARE_OPSTATE(pd, mech_type) \ + ((KCF_TO_PROV_MECHINFO(pd, mech_type).cm_mech_flags) & \ + CRYPTO_CAN_SHARE_OPSTATE) + +/* ps_refcnt is protected by cm_lock in the crypto_minor structure */ +typedef struct crypto_provider_session { + struct crypto_provider_session *ps_next; + crypto_session_id_t ps_session; + kcf_provider_desc_t *ps_provider; + kcf_provider_desc_t *ps_real_provider; + uint_t ps_refcnt; +} crypto_provider_session_t; + +typedef struct crypto_session_data { + kmutex_t sd_lock; + kcondvar_t sd_cv; + uint32_t sd_flags; + int sd_pre_approved_amount; + crypto_ctx_t *sd_digest_ctx; + crypto_ctx_t *sd_encr_ctx; + crypto_ctx_t *sd_decr_ctx; + crypto_ctx_t *sd_sign_ctx; + crypto_ctx_t *sd_verify_ctx; + crypto_ctx_t *sd_sign_recover_ctx; + crypto_ctx_t *sd_verify_recover_ctx; + kcf_provider_desc_t *sd_provider; + void *sd_find_init_cookie; + crypto_provider_session_t *sd_provider_session; +} crypto_session_data_t; + +#define CRYPTO_SESSION_IN_USE 0x00000001 +#define CRYPTO_SESSION_IS_BUSY 0x00000002 +#define CRYPTO_SESSION_IS_CLOSED 0x00000004 + +#define KCF_MAX_PIN_LEN 1024 + +/* + * Per-minor info. + * + * cm_lock protects everything in this structure except for cm_refcnt. + */ +typedef struct crypto_minor { + uint_t cm_refcnt; + kmutex_t cm_lock; + kcondvar_t cm_cv; + crypto_session_data_t **cm_session_table; + uint_t cm_session_table_count; + kcf_provider_desc_t **cm_provider_array; + uint_t cm_provider_count; + crypto_provider_session_t *cm_provider_session; +} crypto_minor_t; + +/* + * Return codes for internal functions + */ +#define KCF_SUCCESS 0x0 /* Successful call */ +#define KCF_INVALID_MECH_NUMBER 0x1 /* invalid mechanism number */ +#define KCF_INVALID_MECH_NAME 0x2 /* invalid mechanism name */ +#define KCF_INVALID_MECH_CLASS 0x3 /* invalid mechanism class */ +#define KCF_MECH_TAB_FULL 0x4 /* Need more room in the mech tabs. */ +#define KCF_INVALID_INDX ((ushort_t)-1) + +/* + * kCF internal mechanism and function group for tracking RNG providers. + */ +#define SUN_RANDOM "random" +#define CRYPTO_FG_RANDOM 0x80000000 /* generate_random() */ + +/* + * Wrappers for ops vectors. In the wrapper definitions below, the pd + * argument always corresponds to a pointer to a provider descriptor + * of type kcf_prov_desc_t. + */ + +#define KCF_PROV_CONTROL_OPS(pd) ((pd)->pd_ops_vector->co_control_ops) +#define KCF_PROV_CTX_OPS(pd) ((pd)->pd_ops_vector->co_ctx_ops) +#define KCF_PROV_DIGEST_OPS(pd) ((pd)->pd_ops_vector->co_digest_ops) +#define KCF_PROV_CIPHER_OPS(pd) ((pd)->pd_ops_vector->co_cipher_ops) +#define KCF_PROV_MAC_OPS(pd) ((pd)->pd_ops_vector->co_mac_ops) +#define KCF_PROV_SIGN_OPS(pd) ((pd)->pd_ops_vector->co_sign_ops) +#define KCF_PROV_VERIFY_OPS(pd) ((pd)->pd_ops_vector->co_verify_ops) +#define KCF_PROV_DUAL_OPS(pd) ((pd)->pd_ops_vector->co_dual_ops) +#define KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) \ + ((pd)->pd_ops_vector->co_dual_cipher_mac_ops) +#define KCF_PROV_RANDOM_OPS(pd) ((pd)->pd_ops_vector->co_random_ops) +#define KCF_PROV_SESSION_OPS(pd) ((pd)->pd_ops_vector->co_session_ops) +#define KCF_PROV_OBJECT_OPS(pd) ((pd)->pd_ops_vector->co_object_ops) +#define KCF_PROV_KEY_OPS(pd) ((pd)->pd_ops_vector->co_key_ops) +#define KCF_PROV_PROVIDER_OPS(pd) ((pd)->pd_ops_vector->co_provider_ops) +#define KCF_PROV_MECH_OPS(pd) ((pd)->pd_ops_vector->co_mech_ops) +#define KCF_PROV_NOSTORE_KEY_OPS(pd) \ + ((pd)->pd_ops_vector->co_nostore_key_ops) + +/* + * Wrappers for crypto_control_ops(9S) entry points. + */ + +#define KCF_PROV_STATUS(pd, status) ( \ + (KCF_PROV_CONTROL_OPS(pd) && \ + KCF_PROV_CONTROL_OPS(pd)->provider_status) ? \ + KCF_PROV_CONTROL_OPS(pd)->provider_status( \ + (pd)->pd_prov_handle, status) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_ctx_ops(9S) entry points. + */ + +#define KCF_PROV_CREATE_CTX_TEMPLATE(pd, mech, key, template, size, req) ( \ + (KCF_PROV_CTX_OPS(pd) && KCF_PROV_CTX_OPS(pd)->create_ctx_template) ? \ + KCF_PROV_CTX_OPS(pd)->create_ctx_template( \ + (pd)->pd_prov_handle, mech, key, template, size, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_FREE_CONTEXT(pd, ctx) ( \ + (KCF_PROV_CTX_OPS(pd) && KCF_PROV_CTX_OPS(pd)->free_context) ? \ + KCF_PROV_CTX_OPS(pd)->free_context(ctx) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_COPYIN_MECH(pd, umech, kmech, errorp, mode) ( \ + (KCF_PROV_MECH_OPS(pd) && KCF_PROV_MECH_OPS(pd)->copyin_mechanism) ? \ + KCF_PROV_MECH_OPS(pd)->copyin_mechanism( \ + (pd)->pd_prov_handle, umech, kmech, errorp, mode) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_COPYOUT_MECH(pd, kmech, umech, errorp, mode) ( \ + (KCF_PROV_MECH_OPS(pd) && KCF_PROV_MECH_OPS(pd)->copyout_mechanism) ? \ + KCF_PROV_MECH_OPS(pd)->copyout_mechanism( \ + (pd)->pd_prov_handle, kmech, umech, errorp, mode) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_FREE_MECH(pd, prov_mech) ( \ + (KCF_PROV_MECH_OPS(pd) && KCF_PROV_MECH_OPS(pd)->free_mechanism) ? \ + KCF_PROV_MECH_OPS(pd)->free_mechanism( \ + (pd)->pd_prov_handle, prov_mech) : CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_digest_ops(9S) entry points. + */ + +#define KCF_PROV_DIGEST_INIT(pd, ctx, mech, req) ( \ + (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_init) ? \ + KCF_PROV_DIGEST_OPS(pd)->digest_init(ctx, mech, req) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * The _ (underscore) in _digest is needed to avoid replacing the + * function digest(). + */ +#define KCF_PROV_DIGEST(pd, ctx, data, _digest, req) ( \ + (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest) ? \ + KCF_PROV_DIGEST_OPS(pd)->digest(ctx, data, _digest, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DIGEST_UPDATE(pd, ctx, data, req) ( \ + (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_update) ? \ + KCF_PROV_DIGEST_OPS(pd)->digest_update(ctx, data, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DIGEST_KEY(pd, ctx, key, req) ( \ + (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_key) ? \ + KCF_PROV_DIGEST_OPS(pd)->digest_key(ctx, key, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DIGEST_FINAL(pd, ctx, digest, req) ( \ + (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_final) ? \ + KCF_PROV_DIGEST_OPS(pd)->digest_final(ctx, digest, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DIGEST_ATOMIC(pd, session, mech, data, digest, req) ( \ + (KCF_PROV_DIGEST_OPS(pd) && KCF_PROV_DIGEST_OPS(pd)->digest_atomic) ? \ + KCF_PROV_DIGEST_OPS(pd)->digest_atomic( \ + (pd)->pd_prov_handle, session, mech, data, digest, req) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_cipher_ops(9S) entry points. + */ + +#define KCF_PROV_ENCRYPT_INIT(pd, ctx, mech, key, template, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_init) ? \ + KCF_PROV_CIPHER_OPS(pd)->encrypt_init(ctx, mech, key, template, \ + req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_ENCRYPT(pd, ctx, plaintext, ciphertext, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt) ? \ + KCF_PROV_CIPHER_OPS(pd)->encrypt(ctx, plaintext, ciphertext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_ENCRYPT_UPDATE(pd, ctx, plaintext, ciphertext, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_update) ? \ + KCF_PROV_CIPHER_OPS(pd)->encrypt_update(ctx, plaintext, \ + ciphertext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_ENCRYPT_FINAL(pd, ctx, ciphertext, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_final) ? \ + KCF_PROV_CIPHER_OPS(pd)->encrypt_final(ctx, ciphertext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_ENCRYPT_ATOMIC(pd, session, mech, key, plaintext, ciphertext, \ + template, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->encrypt_atomic) ? \ + KCF_PROV_CIPHER_OPS(pd)->encrypt_atomic( \ + (pd)->pd_prov_handle, session, mech, key, plaintext, ciphertext, \ + template, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DECRYPT_INIT(pd, ctx, mech, key, template, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt_init) ? \ + KCF_PROV_CIPHER_OPS(pd)->decrypt_init(ctx, mech, key, template, \ + req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DECRYPT(pd, ctx, ciphertext, plaintext, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt) ? \ + KCF_PROV_CIPHER_OPS(pd)->decrypt(ctx, ciphertext, plaintext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DECRYPT_UPDATE(pd, ctx, ciphertext, plaintext, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt_update) ? \ + KCF_PROV_CIPHER_OPS(pd)->decrypt_update(ctx, ciphertext, \ + plaintext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DECRYPT_FINAL(pd, ctx, plaintext, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt_final) ? \ + KCF_PROV_CIPHER_OPS(pd)->decrypt_final(ctx, plaintext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DECRYPT_ATOMIC(pd, session, mech, key, ciphertext, plaintext, \ + template, req) ( \ + (KCF_PROV_CIPHER_OPS(pd) && KCF_PROV_CIPHER_OPS(pd)->decrypt_atomic) ? \ + KCF_PROV_CIPHER_OPS(pd)->decrypt_atomic( \ + (pd)->pd_prov_handle, session, mech, key, ciphertext, plaintext, \ + template, req) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_mac_ops(9S) entry points. + */ + +#define KCF_PROV_MAC_INIT(pd, ctx, mech, key, template, req) ( \ + (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_init) ? \ + KCF_PROV_MAC_OPS(pd)->mac_init(ctx, mech, key, template, req) \ + : CRYPTO_NOT_SUPPORTED) + +/* + * The _ (underscore) in _mac is needed to avoid replacing the + * function mac(). + */ +#define KCF_PROV_MAC(pd, ctx, data, _mac, req) ( \ + (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac) ? \ + KCF_PROV_MAC_OPS(pd)->mac(ctx, data, _mac, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_UPDATE(pd, ctx, data, req) ( \ + (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_update) ? \ + KCF_PROV_MAC_OPS(pd)->mac_update(ctx, data, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_FINAL(pd, ctx, mac, req) ( \ + (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_final) ? \ + KCF_PROV_MAC_OPS(pd)->mac_final(ctx, mac, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_ATOMIC(pd, session, mech, key, data, mac, template, \ + req) ( \ + (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_atomic) ? \ + KCF_PROV_MAC_OPS(pd)->mac_atomic( \ + (pd)->pd_prov_handle, session, mech, key, data, mac, template, \ + req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_VERIFY_ATOMIC(pd, session, mech, key, data, mac, \ + template, req) ( \ + (KCF_PROV_MAC_OPS(pd) && KCF_PROV_MAC_OPS(pd)->mac_verify_atomic) ? \ + KCF_PROV_MAC_OPS(pd)->mac_verify_atomic( \ + (pd)->pd_prov_handle, session, mech, key, data, mac, template, \ + req) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_sign_ops(9S) entry points. + */ + +#define KCF_PROV_SIGN_INIT(pd, ctx, mech, key, template, req) ( \ + (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_init) ? \ + KCF_PROV_SIGN_OPS(pd)->sign_init( \ + ctx, mech, key, template, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SIGN(pd, ctx, data, sig, req) ( \ + (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign) ? \ + KCF_PROV_SIGN_OPS(pd)->sign(ctx, data, sig, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SIGN_UPDATE(pd, ctx, data, req) ( \ + (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_update) ? \ + KCF_PROV_SIGN_OPS(pd)->sign_update(ctx, data, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SIGN_FINAL(pd, ctx, sig, req) ( \ + (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_final) ? \ + KCF_PROV_SIGN_OPS(pd)->sign_final(ctx, sig, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SIGN_ATOMIC(pd, session, mech, key, data, template, \ + sig, req) ( \ + (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_atomic) ? \ + KCF_PROV_SIGN_OPS(pd)->sign_atomic( \ + (pd)->pd_prov_handle, session, mech, key, data, sig, template, \ + req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SIGN_RECOVER_INIT(pd, ctx, mech, key, template, \ + req) ( \ + (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_recover_init) ? \ + KCF_PROV_SIGN_OPS(pd)->sign_recover_init(ctx, mech, key, template, \ + req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SIGN_RECOVER(pd, ctx, data, sig, req) ( \ + (KCF_PROV_SIGN_OPS(pd) && KCF_PROV_SIGN_OPS(pd)->sign_recover) ? \ + KCF_PROV_SIGN_OPS(pd)->sign_recover(ctx, data, sig, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SIGN_RECOVER_ATOMIC(pd, session, mech, key, data, template, \ + sig, req) ( \ + (KCF_PROV_SIGN_OPS(pd) && \ + KCF_PROV_SIGN_OPS(pd)->sign_recover_atomic) ? \ + KCF_PROV_SIGN_OPS(pd)->sign_recover_atomic( \ + (pd)->pd_prov_handle, session, mech, key, data, sig, template, \ + req) : CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_verify_ops(9S) entry points. + */ + +#define KCF_PROV_VERIFY_INIT(pd, ctx, mech, key, template, req) ( \ + (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_init) ? \ + KCF_PROV_VERIFY_OPS(pd)->verify_init(ctx, mech, key, template, \ + req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_VERIFY(pd, ctx, data, sig, req) ( \ + (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->do_verify) ? \ + KCF_PROV_VERIFY_OPS(pd)->do_verify(ctx, data, sig, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_VERIFY_UPDATE(pd, ctx, data, req) ( \ + (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_update) ? \ + KCF_PROV_VERIFY_OPS(pd)->verify_update(ctx, data, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_VERIFY_FINAL(pd, ctx, sig, req) ( \ + (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_final) ? \ + KCF_PROV_VERIFY_OPS(pd)->verify_final(ctx, sig, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_VERIFY_ATOMIC(pd, session, mech, key, data, template, sig, \ + req) ( \ + (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_atomic) ? \ + KCF_PROV_VERIFY_OPS(pd)->verify_atomic( \ + (pd)->pd_prov_handle, session, mech, key, data, sig, template, \ + req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_VERIFY_RECOVER_INIT(pd, ctx, mech, key, template, \ + req) ( \ + (KCF_PROV_VERIFY_OPS(pd) && \ + KCF_PROV_VERIFY_OPS(pd)->verify_recover_init) ? \ + KCF_PROV_VERIFY_OPS(pd)->verify_recover_init(ctx, mech, key, \ + template, req) : CRYPTO_NOT_SUPPORTED) + +/* verify_recover() CSPI routine has different argument order than verify() */ +#define KCF_PROV_VERIFY_RECOVER(pd, ctx, sig, data, req) ( \ + (KCF_PROV_VERIFY_OPS(pd) && KCF_PROV_VERIFY_OPS(pd)->verify_recover) ? \ + KCF_PROV_VERIFY_OPS(pd)->verify_recover(ctx, sig, data, req) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * verify_recover_atomic() CSPI routine has different argument order + * than verify_atomic(). + */ +#define KCF_PROV_VERIFY_RECOVER_ATOMIC(pd, session, mech, key, sig, \ + template, data, req) ( \ + (KCF_PROV_VERIFY_OPS(pd) && \ + KCF_PROV_VERIFY_OPS(pd)->verify_recover_atomic) ? \ + KCF_PROV_VERIFY_OPS(pd)->verify_recover_atomic( \ + (pd)->pd_prov_handle, session, mech, key, sig, data, template, \ + req) : CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_dual_ops(9S) entry points. + */ + +#define KCF_PROV_DIGEST_ENCRYPT_UPDATE(digest_ctx, encrypt_ctx, plaintext, \ + ciphertext, req) ( \ + (KCF_PROV_DUAL_OPS(pd) && \ + KCF_PROV_DUAL_OPS(pd)->digest_encrypt_update) ? \ + KCF_PROV_DUAL_OPS(pd)->digest_encrypt_update( \ + digest_ctx, encrypt_ctx, plaintext, ciphertext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DECRYPT_DIGEST_UPDATE(decrypt_ctx, digest_ctx, ciphertext, \ + plaintext, req) ( \ + (KCF_PROV_DUAL_OPS(pd) && \ + KCF_PROV_DUAL_OPS(pd)->decrypt_digest_update) ? \ + KCF_PROV_DUAL_OPS(pd)->decrypt_digest_update( \ + decrypt_ctx, digest_ctx, ciphertext, plaintext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SIGN_ENCRYPT_UPDATE(sign_ctx, encrypt_ctx, plaintext, \ + ciphertext, req) ( \ + (KCF_PROV_DUAL_OPS(pd) && \ + KCF_PROV_DUAL_OPS(pd)->sign_encrypt_update) ? \ + KCF_PROV_DUAL_OPS(pd)->sign_encrypt_update( \ + sign_ctx, encrypt_ctx, plaintext, ciphertext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_DECRYPT_VERIFY_UPDATE(decrypt_ctx, verify_ctx, ciphertext, \ + plaintext, req) ( \ + (KCF_PROV_DUAL_OPS(pd) && \ + KCF_PROV_DUAL_OPS(pd)->decrypt_verify_update) ? \ + KCF_PROV_DUAL_OPS(pd)->decrypt_verify_update( \ + decrypt_ctx, verify_ctx, ciphertext, plaintext, req) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_dual_cipher_mac_ops(9S) entry points. + */ + +#define KCF_PROV_ENCRYPT_MAC_INIT(pd, ctx, encr_mech, encr_key, mac_mech, \ + mac_key, encr_ctx_template, mac_ctx_template, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_init) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_init( \ + ctx, encr_mech, encr_key, mac_mech, mac_key, encr_ctx_template, \ + mac_ctx_template, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_ENCRYPT_MAC(pd, ctx, plaintext, ciphertext, mac, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac( \ + ctx, plaintext, ciphertext, mac, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_ENCRYPT_MAC_UPDATE(pd, ctx, plaintext, ciphertext, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_update) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_update( \ + ctx, plaintext, ciphertext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_ENCRYPT_MAC_FINAL(pd, ctx, ciphertext, mac, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_final) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_final( \ + ctx, ciphertext, mac, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_ENCRYPT_MAC_ATOMIC(pd, session, encr_mech, encr_key, \ + mac_mech, mac_key, plaintext, ciphertext, mac, \ + encr_ctx_template, mac_ctx_template, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_atomic) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->encrypt_mac_atomic( \ + (pd)->pd_prov_handle, session, encr_mech, encr_key, \ + mac_mech, mac_key, plaintext, ciphertext, mac, \ + encr_ctx_template, mac_ctx_template, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_DECRYPT_INIT(pd, ctx, mac_mech, mac_key, decr_mech, \ + decr_key, mac_ctx_template, decr_ctx_template, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_init) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_init( \ + ctx, mac_mech, mac_key, decr_mech, decr_key, mac_ctx_template, \ + decr_ctx_template, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_DECRYPT(pd, ctx, ciphertext, mac, plaintext, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt( \ + ctx, ciphertext, mac, plaintext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_DECRYPT_UPDATE(pd, ctx, ciphertext, plaintext, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_update) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_update( \ + ctx, ciphertext, plaintext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_DECRYPT_FINAL(pd, ctx, mac, plaintext, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_final) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_final( \ + ctx, mac, plaintext, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_DECRYPT_ATOMIC(pd, session, mac_mech, mac_key, \ + decr_mech, decr_key, ciphertext, mac, plaintext, \ + mac_ctx_template, decr_ctx_template, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_atomic) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_decrypt_atomic( \ + (pd)->pd_prov_handle, session, mac_mech, mac_key, \ + decr_mech, decr_key, ciphertext, mac, plaintext, \ + mac_ctx_template, decr_ctx_template, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_MAC_VERIFY_DECRYPT_ATOMIC(pd, session, mac_mech, mac_key, \ + decr_mech, decr_key, ciphertext, mac, plaintext, \ + mac_ctx_template, decr_ctx_template, req) ( \ + (KCF_PROV_DUAL_CIPHER_MAC_OPS(pd) && \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_verify_decrypt_atomic \ + != NULL) ? \ + KCF_PROV_DUAL_CIPHER_MAC_OPS(pd)->mac_verify_decrypt_atomic( \ + (pd)->pd_prov_handle, session, mac_mech, mac_key, \ + decr_mech, decr_key, ciphertext, mac, plaintext, \ + mac_ctx_template, decr_ctx_template, req) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_random_number_ops(9S) entry points. + */ + +#define KCF_PROV_SEED_RANDOM(pd, session, buf, len, est, flags, req) ( \ + (KCF_PROV_RANDOM_OPS(pd) && KCF_PROV_RANDOM_OPS(pd)->seed_random) ? \ + KCF_PROV_RANDOM_OPS(pd)->seed_random((pd)->pd_prov_handle, \ + session, buf, len, est, flags, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_GENERATE_RANDOM(pd, session, buf, len, req) ( \ + (KCF_PROV_RANDOM_OPS(pd) && \ + KCF_PROV_RANDOM_OPS(pd)->generate_random) ? \ + KCF_PROV_RANDOM_OPS(pd)->generate_random((pd)->pd_prov_handle, \ + session, buf, len, req) : CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_session_ops(9S) entry points. + * + * ops_pd is the provider descriptor that supplies the ops_vector. + * pd is the descriptor that supplies the provider handle. + * Only session open/close needs two handles. + */ + +#define KCF_PROV_SESSION_OPEN(ops_pd, session, req, pd) ( \ + (KCF_PROV_SESSION_OPS(ops_pd) && \ + KCF_PROV_SESSION_OPS(ops_pd)->session_open) ? \ + KCF_PROV_SESSION_OPS(ops_pd)->session_open((pd)->pd_prov_handle, \ + session, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SESSION_CLOSE(ops_pd, session, req, pd) ( \ + (KCF_PROV_SESSION_OPS(ops_pd) && \ + KCF_PROV_SESSION_OPS(ops_pd)->session_close) ? \ + KCF_PROV_SESSION_OPS(ops_pd)->session_close((pd)->pd_prov_handle, \ + session, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SESSION_LOGIN(pd, session, user_type, pin, len, req) ( \ + (KCF_PROV_SESSION_OPS(pd) && \ + KCF_PROV_SESSION_OPS(pd)->session_login) ? \ + KCF_PROV_SESSION_OPS(pd)->session_login((pd)->pd_prov_handle, \ + session, user_type, pin, len, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SESSION_LOGOUT(pd, session, req) ( \ + (KCF_PROV_SESSION_OPS(pd) && \ + KCF_PROV_SESSION_OPS(pd)->session_logout) ? \ + KCF_PROV_SESSION_OPS(pd)->session_logout((pd)->pd_prov_handle, \ + session, req) : CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_object_ops(9S) entry points. + */ + +#define KCF_PROV_OBJECT_CREATE(pd, session, template, count, object, req) ( \ + (KCF_PROV_OBJECT_OPS(pd) && KCF_PROV_OBJECT_OPS(pd)->object_create) ? \ + KCF_PROV_OBJECT_OPS(pd)->object_create((pd)->pd_prov_handle, \ + session, template, count, object, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_OBJECT_COPY(pd, session, object, template, count, \ + new_object, req) ( \ + (KCF_PROV_OBJECT_OPS(pd) && KCF_PROV_OBJECT_OPS(pd)->object_copy) ? \ + KCF_PROV_OBJECT_OPS(pd)->object_copy((pd)->pd_prov_handle, \ + session, object, template, count, new_object, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_OBJECT_DESTROY(pd, session, object, req) ( \ + (KCF_PROV_OBJECT_OPS(pd) && KCF_PROV_OBJECT_OPS(pd)->object_destroy) ? \ + KCF_PROV_OBJECT_OPS(pd)->object_destroy((pd)->pd_prov_handle, \ + session, object, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_OBJECT_GET_SIZE(pd, session, object, size, req) ( \ + (KCF_PROV_OBJECT_OPS(pd) && \ + KCF_PROV_OBJECT_OPS(pd)->object_get_size) ? \ + KCF_PROV_OBJECT_OPS(pd)->object_get_size((pd)->pd_prov_handle, \ + session, object, size, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_OBJECT_GET_ATTRIBUTE_VALUE(pd, session, object, template, \ + count, req) ( \ + (KCF_PROV_OBJECT_OPS(pd) && \ + KCF_PROV_OBJECT_OPS(pd)->object_get_attribute_value) ? \ + KCF_PROV_OBJECT_OPS(pd)->object_get_attribute_value( \ + (pd)->pd_prov_handle, session, object, template, count, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_OBJECT_SET_ATTRIBUTE_VALUE(pd, session, object, template, \ + count, req) ( \ + (KCF_PROV_OBJECT_OPS(pd) && \ + KCF_PROV_OBJECT_OPS(pd)->object_set_attribute_value) ? \ + KCF_PROV_OBJECT_OPS(pd)->object_set_attribute_value( \ + (pd)->pd_prov_handle, session, object, template, count, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_OBJECT_FIND_INIT(pd, session, template, count, ppriv, \ + req) ( \ + (KCF_PROV_OBJECT_OPS(pd) && \ + KCF_PROV_OBJECT_OPS(pd)->object_find_init) ? \ + KCF_PROV_OBJECT_OPS(pd)->object_find_init((pd)->pd_prov_handle, \ + session, template, count, ppriv, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_OBJECT_FIND(pd, ppriv, objects, max_objects, object_count, \ + req) ( \ + (KCF_PROV_OBJECT_OPS(pd) && KCF_PROV_OBJECT_OPS(pd)->object_find) ? \ + KCF_PROV_OBJECT_OPS(pd)->object_find( \ + (pd)->pd_prov_handle, ppriv, objects, max_objects, object_count, \ + req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_OBJECT_FIND_FINAL(pd, ppriv, req) ( \ + (KCF_PROV_OBJECT_OPS(pd) && \ + KCF_PROV_OBJECT_OPS(pd)->object_find_final) ? \ + KCF_PROV_OBJECT_OPS(pd)->object_find_final( \ + (pd)->pd_prov_handle, ppriv, req) : CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_key_ops(9S) entry points. + */ + +#define KCF_PROV_KEY_GENERATE(pd, session, mech, template, count, object, \ + req) ( \ + (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_generate) ? \ + KCF_PROV_KEY_OPS(pd)->key_generate((pd)->pd_prov_handle, \ + session, mech, template, count, object, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_KEY_GENERATE_PAIR(pd, session, mech, pub_template, \ + pub_count, priv_template, priv_count, pub_key, priv_key, req) ( \ + (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_generate_pair) ? \ + KCF_PROV_KEY_OPS(pd)->key_generate_pair((pd)->pd_prov_handle, \ + session, mech, pub_template, pub_count, priv_template, \ + priv_count, pub_key, priv_key, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_KEY_WRAP(pd, session, mech, wrapping_key, key, wrapped_key, \ + wrapped_key_len, req) ( \ + (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_wrap) ? \ + KCF_PROV_KEY_OPS(pd)->key_wrap((pd)->pd_prov_handle, \ + session, mech, wrapping_key, key, wrapped_key, wrapped_key_len, \ + req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_KEY_UNWRAP(pd, session, mech, unwrapping_key, wrapped_key, \ + wrapped_key_len, template, count, key, req) ( \ + (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_unwrap) ? \ + KCF_PROV_KEY_OPS(pd)->key_unwrap((pd)->pd_prov_handle, \ + session, mech, unwrapping_key, wrapped_key, wrapped_key_len, \ + template, count, key, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_KEY_DERIVE(pd, session, mech, base_key, template, count, \ + key, req) ( \ + (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_derive) ? \ + KCF_PROV_KEY_OPS(pd)->key_derive((pd)->pd_prov_handle, \ + session, mech, base_key, template, count, key, req) : \ + CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_KEY_CHECK(pd, mech, key) ( \ + (KCF_PROV_KEY_OPS(pd) && KCF_PROV_KEY_OPS(pd)->key_check) ? \ + KCF_PROV_KEY_OPS(pd)->key_check((pd)->pd_prov_handle, mech, key) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_provider_management_ops(9S) entry points. + * + * ops_pd is the provider descriptor that supplies the ops_vector. + * pd is the descriptor that supplies the provider handle. + * Only ext_info needs two handles. + */ + +#define KCF_PROV_EXT_INFO(ops_pd, provext_info, req, pd) ( \ + (KCF_PROV_PROVIDER_OPS(ops_pd) && \ + KCF_PROV_PROVIDER_OPS(ops_pd)->ext_info) ? \ + KCF_PROV_PROVIDER_OPS(ops_pd)->ext_info((pd)->pd_prov_handle, \ + provext_info, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_INIT_TOKEN(pd, pin, pin_len, label, req) ( \ + (KCF_PROV_PROVIDER_OPS(pd) && KCF_PROV_PROVIDER_OPS(pd)->init_token) ? \ + KCF_PROV_PROVIDER_OPS(pd)->init_token((pd)->pd_prov_handle, \ + pin, pin_len, label, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_INIT_PIN(pd, session, pin, pin_len, req) ( \ + (KCF_PROV_PROVIDER_OPS(pd) && KCF_PROV_PROVIDER_OPS(pd)->init_pin) ? \ + KCF_PROV_PROVIDER_OPS(pd)->init_pin((pd)->pd_prov_handle, \ + session, pin, pin_len, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_SET_PIN(pd, session, old_pin, old_len, new_pin, new_len, \ + req) ( \ + (KCF_PROV_PROVIDER_OPS(pd) && KCF_PROV_PROVIDER_OPS(pd)->set_pin) ? \ + KCF_PROV_PROVIDER_OPS(pd)->set_pin((pd)->pd_prov_handle, \ + session, old_pin, old_len, new_pin, new_len, req) : \ + CRYPTO_NOT_SUPPORTED) + +/* + * Wrappers for crypto_nostore_key_ops(9S) entry points. + */ + +#define KCF_PROV_NOSTORE_KEY_GENERATE(pd, session, mech, template, count, \ + out_template, out_count, req) ( \ + (KCF_PROV_NOSTORE_KEY_OPS(pd) && \ + KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_generate) ? \ + KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_generate( \ + (pd)->pd_prov_handle, session, mech, template, count, \ + out_template, out_count, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_NOSTORE_KEY_GENERATE_PAIR(pd, session, mech, pub_template, \ + pub_count, priv_template, priv_count, out_pub_template, \ + out_pub_count, out_priv_template, out_priv_count, req) ( \ + (KCF_PROV_NOSTORE_KEY_OPS(pd) && \ + KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_generate_pair) ? \ + KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_generate_pair( \ + (pd)->pd_prov_handle, session, mech, pub_template, pub_count, \ + priv_template, priv_count, out_pub_template, out_pub_count, \ + out_priv_template, out_priv_count, req) : CRYPTO_NOT_SUPPORTED) + +#define KCF_PROV_NOSTORE_KEY_DERIVE(pd, session, mech, base_key, template, \ + count, out_template, out_count, req) ( \ + (KCF_PROV_NOSTORE_KEY_OPS(pd) && \ + KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_derive) ? \ + KCF_PROV_NOSTORE_KEY_OPS(pd)->nostore_key_derive( \ + (pd)->pd_prov_handle, session, mech, base_key, template, count, \ + out_template, out_count, req) : CRYPTO_NOT_SUPPORTED) + +/* + * The following routines are exported by the kcf module (/kernel/misc/kcf) + * to the crypto and cryptoadmin modules. + */ + +/* Digest/mac/cipher entry points that take a provider descriptor and session */ +extern int crypto_digest_single(crypto_context_t, crypto_data_t *, + crypto_data_t *, crypto_call_req_t *); + +extern int crypto_mac_single(crypto_context_t, crypto_data_t *, + crypto_data_t *, crypto_call_req_t *); + +extern int crypto_encrypt_single(crypto_context_t, crypto_data_t *, + crypto_data_t *, crypto_call_req_t *); + +extern int crypto_decrypt_single(crypto_context_t, crypto_data_t *, + crypto_data_t *, crypto_call_req_t *); + + +/* Other private digest/mac/cipher entry points not exported through k-API */ +extern int crypto_digest_key_prov(crypto_context_t, crypto_key_t *, + crypto_call_req_t *); + +/* Private sign entry points exported by KCF */ +extern int crypto_sign_single(crypto_context_t, crypto_data_t *, + crypto_data_t *, crypto_call_req_t *); + +extern int crypto_sign_recover_single(crypto_context_t, crypto_data_t *, + crypto_data_t *, crypto_call_req_t *); + +/* Private verify entry points exported by KCF */ +extern int crypto_verify_single(crypto_context_t, crypto_data_t *, + crypto_data_t *, crypto_call_req_t *); + +extern int crypto_verify_recover_single(crypto_context_t, crypto_data_t *, + crypto_data_t *, crypto_call_req_t *); + +/* Private dual operations entry points exported by KCF */ +extern int crypto_digest_encrypt_update(crypto_context_t, crypto_context_t, + crypto_data_t *, crypto_data_t *, crypto_call_req_t *); +extern int crypto_decrypt_digest_update(crypto_context_t, crypto_context_t, + crypto_data_t *, crypto_data_t *, crypto_call_req_t *); +extern int crypto_sign_encrypt_update(crypto_context_t, crypto_context_t, + crypto_data_t *, crypto_data_t *, crypto_call_req_t *); +extern int crypto_decrypt_verify_update(crypto_context_t, crypto_context_t, + crypto_data_t *, crypto_data_t *, crypto_call_req_t *); + +/* Random Number Generation */ +int crypto_seed_random(crypto_provider_handle_t provider, uchar_t *buf, + size_t len, crypto_call_req_t *req); +int crypto_generate_random(crypto_provider_handle_t provider, uchar_t *buf, + size_t len, crypto_call_req_t *req); + +/* Provider Management */ +int crypto_get_provider_info(crypto_provider_id_t id, + crypto_provider_info_t **info, crypto_call_req_t *req); +int crypto_get_provider_mechanisms(crypto_minor_t *, crypto_provider_id_t id, + uint_t *count, crypto_mech_name_t **list); +int crypto_init_token(crypto_provider_handle_t provider, char *pin, + size_t pin_len, char *label, crypto_call_req_t *); +int crypto_init_pin(crypto_provider_handle_t provider, char *pin, + size_t pin_len, crypto_call_req_t *req); +int crypto_set_pin(crypto_provider_handle_t provider, char *old_pin, + size_t old_len, char *new_pin, size_t new_len, crypto_call_req_t *req); +void crypto_free_provider_list(crypto_provider_entry_t *list, uint_t count); +void crypto_free_provider_info(crypto_provider_info_t *info); + +/* Administrative */ +int crypto_get_dev_list(uint_t *count, crypto_dev_list_entry_t **list); +int crypto_get_soft_list(uint_t *count, char **list, size_t *len); +int crypto_get_dev_info(char *name, uint_t instance, uint_t *count, + crypto_mech_name_t **list); +int crypto_get_soft_info(caddr_t name, uint_t *count, + crypto_mech_name_t **list); +int crypto_load_dev_disabled(char *name, uint_t instance, uint_t count, + crypto_mech_name_t *list); +int crypto_load_soft_disabled(caddr_t name, uint_t count, + crypto_mech_name_t *list); +int crypto_unload_soft_module(caddr_t path); +int crypto_load_soft_config(caddr_t name, uint_t count, + crypto_mech_name_t *list); +int crypto_load_door(uint_t did); +void crypto_free_mech_list(crypto_mech_name_t *list, uint_t count); +void crypto_free_dev_list(crypto_dev_list_entry_t *list, uint_t count); + +/* Miscellaneous */ +int crypto_get_mechanism_number(caddr_t name, crypto_mech_type_t *number); +int crypto_get_function_list(crypto_provider_id_t id, + crypto_function_list_t **list, int kmflag); +void crypto_free_function_list(crypto_function_list_t *list); +int crypto_build_permitted_mech_names(kcf_provider_desc_t *, + crypto_mech_name_t **, uint_t *, int); +extern void kcf_destroy_mech_tabs(void); +extern void kcf_init_mech_tabs(void); +extern int kcf_add_mech_provider(short, kcf_provider_desc_t *, + kcf_prov_mech_desc_t **); +extern void kcf_remove_mech_provider(char *, kcf_provider_desc_t *); +extern int kcf_get_mech_entry(crypto_mech_type_t, kcf_mech_entry_t **); +extern kcf_provider_desc_t *kcf_alloc_provider_desc(crypto_provider_info_t *); +extern void kcf_provider_zero_refcnt(kcf_provider_desc_t *); +extern void kcf_free_provider_desc(kcf_provider_desc_t *); +extern void kcf_soft_config_init(void); +extern int get_sw_provider_for_mech(crypto_mech_name_t, char **); +extern crypto_mech_type_t crypto_mech2id_common(char *, boolean_t); +extern void undo_register_provider(kcf_provider_desc_t *, boolean_t); +extern void redo_register_provider(kcf_provider_desc_t *); +extern void kcf_rnd_init(void); +extern boolean_t kcf_rngprov_check(void); +extern int kcf_rnd_get_pseudo_bytes(uint8_t *, size_t); +extern int kcf_rnd_get_bytes(uint8_t *, size_t, boolean_t, boolean_t); +extern int random_add_pseudo_entropy(uint8_t *, size_t, uint_t); +extern void kcf_rnd_schedule_timeout(boolean_t); +extern int crypto_uio_data(crypto_data_t *, uchar_t *, int, cmd_type_t, + void *, void (*update)(void)); +extern int crypto_mblk_data(crypto_data_t *, uchar_t *, int, cmd_type_t, + void *, void (*update)(void)); +extern int crypto_put_output_data(uchar_t *, crypto_data_t *, int); +extern int crypto_get_input_data(crypto_data_t *, uchar_t **, uchar_t *); +extern int crypto_copy_key_to_ctx(crypto_key_t *, crypto_key_t **, size_t *, + int kmflag); +extern int crypto_digest_data(crypto_data_t *, void *, uchar_t *, + void (*update)(void), void (*final)(void), uchar_t); +extern int crypto_update_iov(void *, crypto_data_t *, crypto_data_t *, + int (*cipher)(void *, caddr_t, size_t, crypto_data_t *), + void (*copy_block)(uint8_t *, uint64_t *)); +extern int crypto_update_uio(void *, crypto_data_t *, crypto_data_t *, + int (*cipher)(void *, caddr_t, size_t, crypto_data_t *), + void (*copy_block)(uint8_t *, uint64_t *)); +extern int crypto_update_mp(void *, crypto_data_t *, crypto_data_t *, + int (*cipher)(void *, caddr_t, size_t, crypto_data_t *), + void (*copy_block)(uint8_t *, uint64_t *)); +extern int crypto_get_key_attr(crypto_key_t *, crypto_attr_type_t, uchar_t **, + ssize_t *); + +/* Access to the provider's table */ +extern void kcf_prov_tab_destroy(void); +extern void kcf_prov_tab_init(void); +extern int kcf_prov_tab_add_provider(kcf_provider_desc_t *); +extern int kcf_prov_tab_rem_provider(crypto_provider_id_t); +extern kcf_provider_desc_t *kcf_prov_tab_lookup_by_name(char *); +extern kcf_provider_desc_t *kcf_prov_tab_lookup_by_dev(char *, uint_t); +extern int kcf_get_hw_prov_tab(uint_t *, kcf_provider_desc_t ***, int, + char *, uint_t, boolean_t); +extern int kcf_get_slot_list(uint_t *, kcf_provider_desc_t ***, boolean_t); +extern void kcf_free_provider_tab(uint_t, kcf_provider_desc_t **); +extern kcf_provider_desc_t *kcf_prov_tab_lookup(crypto_provider_id_t); +extern int kcf_get_sw_prov(crypto_mech_type_t, kcf_provider_desc_t **, + kcf_mech_entry_t **, boolean_t); + +/* Access to the policy table */ +extern boolean_t is_mech_disabled(kcf_provider_desc_t *, crypto_mech_name_t); +extern boolean_t is_mech_disabled_byname(crypto_provider_type_t, char *, + uint_t, crypto_mech_name_t); +extern void kcf_policy_tab_init(void); +extern void kcf_policy_free_desc(kcf_policy_desc_t *); +extern void kcf_policy_remove_by_name(char *, uint_t *, crypto_mech_name_t **); +extern void kcf_policy_remove_by_dev(char *, uint_t, uint_t *, + crypto_mech_name_t **); +extern kcf_policy_desc_t *kcf_policy_lookup_by_name(char *); +extern kcf_policy_desc_t *kcf_policy_lookup_by_dev(char *, uint_t); +extern int kcf_policy_load_soft_disabled(char *, uint_t, crypto_mech_name_t *, + uint_t *, crypto_mech_name_t **); +extern int kcf_policy_load_dev_disabled(char *, uint_t, uint_t, + crypto_mech_name_t *, uint_t *, crypto_mech_name_t **); +extern boolean_t in_soft_config_list(char *); + + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRYPTO_IMPL_H */ diff --git a/module/icp/include/sys/crypto/ioctl.h b/module/icp/include/sys/crypto/ioctl.h new file mode 100644 index 000000000000..dd59ca7f2be5 --- /dev/null +++ b/module/icp/include/sys/crypto/ioctl.h @@ -0,0 +1,1483 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CRYPTO_IOCTL_H +#define _SYS_CRYPTO_IOCTL_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include + +#define CRYPTO_MAX_ATTRIBUTE_COUNT 128 + +#define CRYPTO_IOFLAGS_RW_SESSION 0x00000001 + +#define CRYPTO(x) (('y' << 8) | (x)) + +#define MAX_NUM_THRESHOLD 7 + +/* the PKCS11 Mechanisms */ +#define CKM_RC4 0x00000111 +#define CKM_DES3_ECB 0x00000132 +#define CKM_DES3_CBC 0x00000133 +#define CKM_MD5 0x00000210 +#define CKM_SHA_1 0x00000220 +#define CKM_AES_ECB 0x00001081 +#define CKM_AES_CBC 0x00001082 + +/* + * General Purpose Ioctls + */ + +typedef struct fl_mechs_threshold { + int mech_type; + uint32_t mech_threshold; +} fl_mechs_threshold_t; + +typedef struct crypto_function_list { + boolean_t fl_digest_init; + boolean_t fl_digest; + boolean_t fl_digest_update; + boolean_t fl_digest_key; + boolean_t fl_digest_final; + + boolean_t fl_encrypt_init; + boolean_t fl_encrypt; + boolean_t fl_encrypt_update; + boolean_t fl_encrypt_final; + + boolean_t fl_decrypt_init; + boolean_t fl_decrypt; + boolean_t fl_decrypt_update; + boolean_t fl_decrypt_final; + + boolean_t fl_mac_init; + boolean_t fl_mac; + boolean_t fl_mac_update; + boolean_t fl_mac_final; + + boolean_t fl_sign_init; + boolean_t fl_sign; + boolean_t fl_sign_update; + boolean_t fl_sign_final; + boolean_t fl_sign_recover_init; + boolean_t fl_sign_recover; + + boolean_t fl_verify_init; + boolean_t fl_verify; + boolean_t fl_verify_update; + boolean_t fl_verify_final; + boolean_t fl_verify_recover_init; + boolean_t fl_verify_recover; + + boolean_t fl_digest_encrypt_update; + boolean_t fl_decrypt_digest_update; + boolean_t fl_sign_encrypt_update; + boolean_t fl_decrypt_verify_update; + + boolean_t fl_seed_random; + boolean_t fl_generate_random; + + boolean_t fl_session_open; + boolean_t fl_session_close; + boolean_t fl_session_login; + boolean_t fl_session_logout; + + boolean_t fl_object_create; + boolean_t fl_object_copy; + boolean_t fl_object_destroy; + boolean_t fl_object_get_size; + boolean_t fl_object_get_attribute_value; + boolean_t fl_object_set_attribute_value; + boolean_t fl_object_find_init; + boolean_t fl_object_find; + boolean_t fl_object_find_final; + + boolean_t fl_key_generate; + boolean_t fl_key_generate_pair; + boolean_t fl_key_wrap; + boolean_t fl_key_unwrap; + boolean_t fl_key_derive; + + boolean_t fl_init_token; + boolean_t fl_init_pin; + boolean_t fl_set_pin; + + boolean_t prov_is_limited; + uint32_t prov_hash_threshold; + uint32_t prov_hash_limit; + + int total_threshold_count; + fl_mechs_threshold_t fl_threshold[MAX_NUM_THRESHOLD]; +} crypto_function_list_t; + +typedef struct crypto_get_function_list { + uint_t fl_return_value; + crypto_provider_id_t fl_provider_id; + crypto_function_list_t fl_list; +} crypto_get_function_list_t; + +typedef struct crypto_get_mechanism_number { + uint_t pn_return_value; + caddr_t pn_mechanism_string; + size_t pn_mechanism_len; + crypto_mech_type_t pn_internal_number; +} crypto_get_mechanism_number_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_get_mechanism_number32 { + uint32_t pn_return_value; + caddr32_t pn_mechanism_string; + size32_t pn_mechanism_len; + crypto_mech_type_t pn_internal_number; +} crypto_get_mechanism_number32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_GET_FUNCTION_LIST CRYPTO(20) +#define CRYPTO_GET_MECHANISM_NUMBER CRYPTO(21) + +/* + * Session Ioctls + */ + +typedef uint32_t crypto_flags_t; + +typedef struct crypto_open_session { + uint_t os_return_value; + crypto_session_id_t os_session; + crypto_flags_t os_flags; + crypto_provider_id_t os_provider_id; +} crypto_open_session_t; + +typedef struct crypto_close_session { + uint_t cs_return_value; + crypto_session_id_t cs_session; +} crypto_close_session_t; + +typedef struct crypto_close_all_sessions { + uint_t as_return_value; + crypto_provider_id_t as_provider_id; +} crypto_close_all_sessions_t; + +#define CRYPTO_OPEN_SESSION CRYPTO(30) +#define CRYPTO_CLOSE_SESSION CRYPTO(31) +#define CRYPTO_CLOSE_ALL_SESSIONS CRYPTO(32) + +/* + * Login Ioctls + */ +typedef struct crypto_login { + uint_t co_return_value; + crypto_session_id_t co_session; + uint_t co_user_type; + uint_t co_pin_len; + caddr_t co_pin; +} crypto_login_t; + +typedef struct crypto_logout { + uint_t cl_return_value; + crypto_session_id_t cl_session; +} crypto_logout_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +typedef struct crypto_login32 { + uint32_t co_return_value; + crypto_session_id_t co_session; + uint32_t co_user_type; + uint32_t co_pin_len; + caddr32_t co_pin; +} crypto_login32_t; + +typedef struct crypto_logout32 { + uint32_t cl_return_value; + crypto_session_id_t cl_session; +} crypto_logout32_t; + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_LOGIN CRYPTO(40) +#define CRYPTO_LOGOUT CRYPTO(41) + +/* flag for encrypt and decrypt operations */ +#define CRYPTO_INPLACE_OPERATION 0x00000001 + +/* + * Cryptographic Ioctls + */ +typedef struct crypto_encrypt { + uint_t ce_return_value; + crypto_session_id_t ce_session; + size_t ce_datalen; + caddr_t ce_databuf; + size_t ce_encrlen; + caddr_t ce_encrbuf; + uint_t ce_flags; +} crypto_encrypt_t; + +typedef struct crypto_encrypt_init { + uint_t ei_return_value; + crypto_session_id_t ei_session; + crypto_mechanism_t ei_mech; + crypto_key_t ei_key; +} crypto_encrypt_init_t; + +typedef struct crypto_encrypt_update { + uint_t eu_return_value; + crypto_session_id_t eu_session; + size_t eu_datalen; + caddr_t eu_databuf; + size_t eu_encrlen; + caddr_t eu_encrbuf; +} crypto_encrypt_update_t; + +typedef struct crypto_encrypt_final { + uint_t ef_return_value; + crypto_session_id_t ef_session; + size_t ef_encrlen; + caddr_t ef_encrbuf; +} crypto_encrypt_final_t; + +typedef struct crypto_decrypt { + uint_t cd_return_value; + crypto_session_id_t cd_session; + size_t cd_encrlen; + caddr_t cd_encrbuf; + size_t cd_datalen; + caddr_t cd_databuf; + uint_t cd_flags; +} crypto_decrypt_t; + +typedef struct crypto_decrypt_init { + uint_t di_return_value; + crypto_session_id_t di_session; + crypto_mechanism_t di_mech; + crypto_key_t di_key; +} crypto_decrypt_init_t; + +typedef struct crypto_decrypt_update { + uint_t du_return_value; + crypto_session_id_t du_session; + size_t du_encrlen; + caddr_t du_encrbuf; + size_t du_datalen; + caddr_t du_databuf; +} crypto_decrypt_update_t; + +typedef struct crypto_decrypt_final { + uint_t df_return_value; + crypto_session_id_t df_session; + size_t df_datalen; + caddr_t df_databuf; +} crypto_decrypt_final_t; + +typedef struct crypto_digest { + uint_t cd_return_value; + crypto_session_id_t cd_session; + size_t cd_datalen; + caddr_t cd_databuf; + size_t cd_digestlen; + caddr_t cd_digestbuf; +} crypto_digest_t; + +typedef struct crypto_digest_init { + uint_t di_return_value; + crypto_session_id_t di_session; + crypto_mechanism_t di_mech; +} crypto_digest_init_t; + +typedef struct crypto_digest_update { + uint_t du_return_value; + crypto_session_id_t du_session; + size_t du_datalen; + caddr_t du_databuf; +} crypto_digest_update_t; + +typedef struct crypto_digest_key { + uint_t dk_return_value; + crypto_session_id_t dk_session; + crypto_key_t dk_key; +} crypto_digest_key_t; + +typedef struct crypto_digest_final { + uint_t df_return_value; + crypto_session_id_t df_session; + size_t df_digestlen; + caddr_t df_digestbuf; +} crypto_digest_final_t; + +typedef struct crypto_mac { + uint_t cm_return_value; + crypto_session_id_t cm_session; + size_t cm_datalen; + caddr_t cm_databuf; + size_t cm_maclen; + caddr_t cm_macbuf; +} crypto_mac_t; + +typedef struct crypto_mac_init { + uint_t mi_return_value; + crypto_session_id_t mi_session; + crypto_mechanism_t mi_mech; + crypto_key_t mi_key; +} crypto_mac_init_t; + +typedef struct crypto_mac_update { + uint_t mu_return_value; + crypto_session_id_t mu_session; + size_t mu_datalen; + caddr_t mu_databuf; +} crypto_mac_update_t; + +typedef struct crypto_mac_final { + uint_t mf_return_value; + crypto_session_id_t mf_session; + size_t mf_maclen; + caddr_t mf_macbuf; +} crypto_mac_final_t; + +typedef struct crypto_sign { + uint_t cs_return_value; + crypto_session_id_t cs_session; + size_t cs_datalen; + caddr_t cs_databuf; + size_t cs_signlen; + caddr_t cs_signbuf; +} crypto_sign_t; + +typedef struct crypto_sign_init { + uint_t si_return_value; + crypto_session_id_t si_session; + crypto_mechanism_t si_mech; + crypto_key_t si_key; +} crypto_sign_init_t; + +typedef struct crypto_sign_update { + uint_t su_return_value; + crypto_session_id_t su_session; + size_t su_datalen; + caddr_t su_databuf; +} crypto_sign_update_t; + +typedef struct crypto_sign_final { + uint_t sf_return_value; + crypto_session_id_t sf_session; + size_t sf_signlen; + caddr_t sf_signbuf; +} crypto_sign_final_t; + +typedef struct crypto_sign_recover_init { + uint_t ri_return_value; + crypto_session_id_t ri_session; + crypto_mechanism_t ri_mech; + crypto_key_t ri_key; +} crypto_sign_recover_init_t; + +typedef struct crypto_sign_recover { + uint_t sr_return_value; + crypto_session_id_t sr_session; + size_t sr_datalen; + caddr_t sr_databuf; + size_t sr_signlen; + caddr_t sr_signbuf; +} crypto_sign_recover_t; + +typedef struct crypto_verify { + uint_t cv_return_value; + crypto_session_id_t cv_session; + size_t cv_datalen; + caddr_t cv_databuf; + size_t cv_signlen; + caddr_t cv_signbuf; +} crypto_verify_t; + +typedef struct crypto_verify_init { + uint_t vi_return_value; + crypto_session_id_t vi_session; + crypto_mechanism_t vi_mech; + crypto_key_t vi_key; +} crypto_verify_init_t; + +typedef struct crypto_verify_update { + uint_t vu_return_value; + crypto_session_id_t vu_session; + size_t vu_datalen; + caddr_t vu_databuf; +} crypto_verify_update_t; + +typedef struct crypto_verify_final { + uint_t vf_return_value; + crypto_session_id_t vf_session; + size_t vf_signlen; + caddr_t vf_signbuf; +} crypto_verify_final_t; + +typedef struct crypto_verify_recover_init { + uint_t ri_return_value; + crypto_session_id_t ri_session; + crypto_mechanism_t ri_mech; + crypto_key_t ri_key; +} crypto_verify_recover_init_t; + +typedef struct crypto_verify_recover { + uint_t vr_return_value; + crypto_session_id_t vr_session; + size_t vr_signlen; + caddr_t vr_signbuf; + size_t vr_datalen; + caddr_t vr_databuf; +} crypto_verify_recover_t; + +typedef struct crypto_digest_encrypt_update { + uint_t eu_return_value; + crypto_session_id_t eu_session; + size_t eu_datalen; + caddr_t eu_databuf; + size_t eu_encrlen; + caddr_t eu_encrbuf; +} crypto_digest_encrypt_update_t; + +typedef struct crypto_decrypt_digest_update { + uint_t du_return_value; + crypto_session_id_t du_session; + size_t du_encrlen; + caddr_t du_encrbuf; + size_t du_datalen; + caddr_t du_databuf; +} crypto_decrypt_digest_update_t; + +typedef struct crypto_sign_encrypt_update { + uint_t eu_return_value; + crypto_session_id_t eu_session; + size_t eu_datalen; + caddr_t eu_databuf; + size_t eu_encrlen; + caddr_t eu_encrbuf; +} crypto_sign_encrypt_update_t; + +typedef struct crypto_decrypt_verify_update { + uint_t vu_return_value; + crypto_session_id_t vu_session; + size_t vu_encrlen; + caddr_t vu_encrbuf; + size_t vu_datalen; + caddr_t vu_databuf; +} crypto_decrypt_verify_update_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +typedef struct crypto_encrypt32 { + uint32_t ce_return_value; + crypto_session_id_t ce_session; + size32_t ce_datalen; + caddr32_t ce_databuf; + size32_t ce_encrlen; + caddr32_t ce_encrbuf; + uint32_t ce_flags; +} crypto_encrypt32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_encrypt_init32 { + uint32_t ei_return_value; + crypto_session_id_t ei_session; + crypto_mechanism32_t ei_mech; + crypto_key32_t ei_key; +} crypto_encrypt_init32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +typedef struct crypto_encrypt_update32 { + uint32_t eu_return_value; + crypto_session_id_t eu_session; + size32_t eu_datalen; + caddr32_t eu_databuf; + size32_t eu_encrlen; + caddr32_t eu_encrbuf; +} crypto_encrypt_update32_t; + +typedef struct crypto_encrypt_final32 { + uint32_t ef_return_value; + crypto_session_id_t ef_session; + size32_t ef_encrlen; + caddr32_t ef_encrbuf; +} crypto_encrypt_final32_t; + +typedef struct crypto_decrypt32 { + uint32_t cd_return_value; + crypto_session_id_t cd_session; + size32_t cd_encrlen; + caddr32_t cd_encrbuf; + size32_t cd_datalen; + caddr32_t cd_databuf; + uint32_t cd_flags; +} crypto_decrypt32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_decrypt_init32 { + uint32_t di_return_value; + crypto_session_id_t di_session; + crypto_mechanism32_t di_mech; + crypto_key32_t di_key; +} crypto_decrypt_init32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +typedef struct crypto_decrypt_update32 { + uint32_t du_return_value; + crypto_session_id_t du_session; + size32_t du_encrlen; + caddr32_t du_encrbuf; + size32_t du_datalen; + caddr32_t du_databuf; +} crypto_decrypt_update32_t; + +typedef struct crypto_decrypt_final32 { + uint32_t df_return_value; + crypto_session_id_t df_session; + size32_t df_datalen; + caddr32_t df_databuf; +} crypto_decrypt_final32_t; + +typedef struct crypto_digest32 { + uint32_t cd_return_value; + crypto_session_id_t cd_session; + size32_t cd_datalen; + caddr32_t cd_databuf; + size32_t cd_digestlen; + caddr32_t cd_digestbuf; +} crypto_digest32_t; + +typedef struct crypto_digest_init32 { + uint32_t di_return_value; + crypto_session_id_t di_session; + crypto_mechanism32_t di_mech; +} crypto_digest_init32_t; + +typedef struct crypto_digest_update32 { + uint32_t du_return_value; + crypto_session_id_t du_session; + size32_t du_datalen; + caddr32_t du_databuf; +} crypto_digest_update32_t; + +typedef struct crypto_digest_key32 { + uint32_t dk_return_value; + crypto_session_id_t dk_session; + crypto_key32_t dk_key; +} crypto_digest_key32_t; + +typedef struct crypto_digest_final32 { + uint32_t df_return_value; + crypto_session_id_t df_session; + size32_t df_digestlen; + caddr32_t df_digestbuf; +} crypto_digest_final32_t; + +typedef struct crypto_mac32 { + uint32_t cm_return_value; + crypto_session_id_t cm_session; + size32_t cm_datalen; + caddr32_t cm_databuf; + size32_t cm_maclen; + caddr32_t cm_macbuf; +} crypto_mac32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_mac_init32 { + uint32_t mi_return_value; + crypto_session_id_t mi_session; + crypto_mechanism32_t mi_mech; + crypto_key32_t mi_key; +} crypto_mac_init32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +typedef struct crypto_mac_update32 { + uint32_t mu_return_value; + crypto_session_id_t mu_session; + size32_t mu_datalen; + caddr32_t mu_databuf; +} crypto_mac_update32_t; + +typedef struct crypto_mac_final32 { + uint32_t mf_return_value; + crypto_session_id_t mf_session; + size32_t mf_maclen; + caddr32_t mf_macbuf; +} crypto_mac_final32_t; + +typedef struct crypto_sign32 { + uint32_t cs_return_value; + crypto_session_id_t cs_session; + size32_t cs_datalen; + caddr32_t cs_databuf; + size32_t cs_signlen; + caddr32_t cs_signbuf; +} crypto_sign32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_sign_init32 { + uint32_t si_return_value; + crypto_session_id_t si_session; + crypto_mechanism32_t si_mech; + crypto_key32_t si_key; +} crypto_sign_init32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +typedef struct crypto_sign_update32 { + uint32_t su_return_value; + crypto_session_id_t su_session; + size32_t su_datalen; + caddr32_t su_databuf; +} crypto_sign_update32_t; + +typedef struct crypto_sign_final32 { + uint32_t sf_return_value; + crypto_session_id_t sf_session; + size32_t sf_signlen; + caddr32_t sf_signbuf; +} crypto_sign_final32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_sign_recover_init32 { + uint32_t ri_return_value; + crypto_session_id_t ri_session; + crypto_mechanism32_t ri_mech; + crypto_key32_t ri_key; +} crypto_sign_recover_init32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +typedef struct crypto_sign_recover32 { + uint32_t sr_return_value; + crypto_session_id_t sr_session; + size32_t sr_datalen; + caddr32_t sr_databuf; + size32_t sr_signlen; + caddr32_t sr_signbuf; +} crypto_sign_recover32_t; + +typedef struct crypto_verify32 { + uint32_t cv_return_value; + crypto_session_id_t cv_session; + size32_t cv_datalen; + caddr32_t cv_databuf; + size32_t cv_signlen; + caddr32_t cv_signbuf; +} crypto_verify32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_verify_init32 { + uint32_t vi_return_value; + crypto_session_id_t vi_session; + crypto_mechanism32_t vi_mech; + crypto_key32_t vi_key; +} crypto_verify_init32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +typedef struct crypto_verify_update32 { + uint32_t vu_return_value; + crypto_session_id_t vu_session; + size32_t vu_datalen; + caddr32_t vu_databuf; +} crypto_verify_update32_t; + +typedef struct crypto_verify_final32 { + uint32_t vf_return_value; + crypto_session_id_t vf_session; + size32_t vf_signlen; + caddr32_t vf_signbuf; +} crypto_verify_final32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_verify_recover_init32 { + uint32_t ri_return_value; + crypto_session_id_t ri_session; + crypto_mechanism32_t ri_mech; + crypto_key32_t ri_key; +} crypto_verify_recover_init32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +typedef struct crypto_verify_recover32 { + uint32_t vr_return_value; + crypto_session_id_t vr_session; + size32_t vr_signlen; + caddr32_t vr_signbuf; + size32_t vr_datalen; + caddr32_t vr_databuf; +} crypto_verify_recover32_t; + +typedef struct crypto_digest_encrypt_update32 { + uint32_t eu_return_value; + crypto_session_id_t eu_session; + size32_t eu_datalen; + caddr32_t eu_databuf; + size32_t eu_encrlen; + caddr32_t eu_encrbuf; +} crypto_digest_encrypt_update32_t; + +typedef struct crypto_decrypt_digest_update32 { + uint32_t du_return_value; + crypto_session_id_t du_session; + size32_t du_encrlen; + caddr32_t du_encrbuf; + size32_t du_datalen; + caddr32_t du_databuf; +} crypto_decrypt_digest_update32_t; + +typedef struct crypto_sign_encrypt_update32 { + uint32_t eu_return_value; + crypto_session_id_t eu_session; + size32_t eu_datalen; + caddr32_t eu_databuf; + size32_t eu_encrlen; + caddr32_t eu_encrbuf; +} crypto_sign_encrypt_update32_t; + +typedef struct crypto_decrypt_verify_update32 { + uint32_t vu_return_value; + crypto_session_id_t vu_session; + size32_t vu_encrlen; + caddr32_t vu_encrbuf; + size32_t vu_datalen; + caddr32_t vu_databuf; +} crypto_decrypt_verify_update32_t; + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_ENCRYPT CRYPTO(50) +#define CRYPTO_ENCRYPT_INIT CRYPTO(51) +#define CRYPTO_ENCRYPT_UPDATE CRYPTO(52) +#define CRYPTO_ENCRYPT_FINAL CRYPTO(53) +#define CRYPTO_DECRYPT CRYPTO(54) +#define CRYPTO_DECRYPT_INIT CRYPTO(55) +#define CRYPTO_DECRYPT_UPDATE CRYPTO(56) +#define CRYPTO_DECRYPT_FINAL CRYPTO(57) + +#define CRYPTO_DIGEST CRYPTO(58) +#define CRYPTO_DIGEST_INIT CRYPTO(59) +#define CRYPTO_DIGEST_UPDATE CRYPTO(60) +#define CRYPTO_DIGEST_KEY CRYPTO(61) +#define CRYPTO_DIGEST_FINAL CRYPTO(62) +#define CRYPTO_MAC CRYPTO(63) +#define CRYPTO_MAC_INIT CRYPTO(64) +#define CRYPTO_MAC_UPDATE CRYPTO(65) +#define CRYPTO_MAC_FINAL CRYPTO(66) + +#define CRYPTO_SIGN CRYPTO(67) +#define CRYPTO_SIGN_INIT CRYPTO(68) +#define CRYPTO_SIGN_UPDATE CRYPTO(69) +#define CRYPTO_SIGN_FINAL CRYPTO(70) +#define CRYPTO_SIGN_RECOVER_INIT CRYPTO(71) +#define CRYPTO_SIGN_RECOVER CRYPTO(72) +#define CRYPTO_VERIFY CRYPTO(73) +#define CRYPTO_VERIFY_INIT CRYPTO(74) +#define CRYPTO_VERIFY_UPDATE CRYPTO(75) +#define CRYPTO_VERIFY_FINAL CRYPTO(76) +#define CRYPTO_VERIFY_RECOVER_INIT CRYPTO(77) +#define CRYPTO_VERIFY_RECOVER CRYPTO(78) + +#define CRYPTO_DIGEST_ENCRYPT_UPDATE CRYPTO(79) +#define CRYPTO_DECRYPT_DIGEST_UPDATE CRYPTO(80) +#define CRYPTO_SIGN_ENCRYPT_UPDATE CRYPTO(81) +#define CRYPTO_DECRYPT_VERIFY_UPDATE CRYPTO(82) + +/* + * Random Number Ioctls + */ +typedef struct crypto_seed_random { + uint_t sr_return_value; + crypto_session_id_t sr_session; + size_t sr_seedlen; + caddr_t sr_seedbuf; +} crypto_seed_random_t; + +typedef struct crypto_generate_random { + uint_t gr_return_value; + crypto_session_id_t gr_session; + caddr_t gr_buf; + size_t gr_buflen; +} crypto_generate_random_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +typedef struct crypto_seed_random32 { + uint32_t sr_return_value; + crypto_session_id_t sr_session; + size32_t sr_seedlen; + caddr32_t sr_seedbuf; +} crypto_seed_random32_t; + +typedef struct crypto_generate_random32 { + uint32_t gr_return_value; + crypto_session_id_t gr_session; + caddr32_t gr_buf; + size32_t gr_buflen; +} crypto_generate_random32_t; + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_SEED_RANDOM CRYPTO(90) +#define CRYPTO_GENERATE_RANDOM CRYPTO(91) + +/* + * Object Management Ioctls + */ +typedef struct crypto_object_create { + uint_t oc_return_value; + crypto_session_id_t oc_session; + crypto_object_id_t oc_handle; + uint_t oc_count; + caddr_t oc_attributes; +} crypto_object_create_t; + +typedef struct crypto_object_copy { + uint_t oc_return_value; + crypto_session_id_t oc_session; + crypto_object_id_t oc_handle; + crypto_object_id_t oc_new_handle; + uint_t oc_count; + caddr_t oc_new_attributes; +} crypto_object_copy_t; + +typedef struct crypto_object_destroy { + uint_t od_return_value; + crypto_session_id_t od_session; + crypto_object_id_t od_handle; +} crypto_object_destroy_t; + +typedef struct crypto_object_get_attribute_value { + uint_t og_return_value; + crypto_session_id_t og_session; + crypto_object_id_t og_handle; + uint_t og_count; + caddr_t og_attributes; +} crypto_object_get_attribute_value_t; + +typedef struct crypto_object_get_size { + uint_t gs_return_value; + crypto_session_id_t gs_session; + crypto_object_id_t gs_handle; + size_t gs_size; +} crypto_object_get_size_t; + +typedef struct crypto_object_set_attribute_value { + uint_t sa_return_value; + crypto_session_id_t sa_session; + crypto_object_id_t sa_handle; + uint_t sa_count; + caddr_t sa_attributes; +} crypto_object_set_attribute_value_t; + +typedef struct crypto_object_find_init { + uint_t fi_return_value; + crypto_session_id_t fi_session; + uint_t fi_count; + caddr_t fi_attributes; +} crypto_object_find_init_t; + +typedef struct crypto_object_find_update { + uint_t fu_return_value; + crypto_session_id_t fu_session; + uint_t fu_max_count; + uint_t fu_count; + caddr_t fu_handles; +} crypto_object_find_update_t; + +typedef struct crypto_object_find_final { + uint_t ff_return_value; + crypto_session_id_t ff_session; +} crypto_object_find_final_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +typedef struct crypto_object_create32 { + uint32_t oc_return_value; + crypto_session_id_t oc_session; + crypto_object_id_t oc_handle; + uint32_t oc_count; + caddr32_t oc_attributes; +} crypto_object_create32_t; + +typedef struct crypto_object_copy32 { + uint32_t oc_return_value; + crypto_session_id_t oc_session; + crypto_object_id_t oc_handle; + crypto_object_id_t oc_new_handle; + uint32_t oc_count; + caddr32_t oc_new_attributes; +} crypto_object_copy32_t; + +typedef struct crypto_object_destroy32 { + uint32_t od_return_value; + crypto_session_id_t od_session; + crypto_object_id_t od_handle; +} crypto_object_destroy32_t; + +typedef struct crypto_object_get_attribute_value32 { + uint32_t og_return_value; + crypto_session_id_t og_session; + crypto_object_id_t og_handle; + uint32_t og_count; + caddr32_t og_attributes; +} crypto_object_get_attribute_value32_t; + +typedef struct crypto_object_get_size32 { + uint32_t gs_return_value; + crypto_session_id_t gs_session; + crypto_object_id_t gs_handle; + size32_t gs_size; +} crypto_object_get_size32_t; + +typedef struct crypto_object_set_attribute_value32 { + uint32_t sa_return_value; + crypto_session_id_t sa_session; + crypto_object_id_t sa_handle; + uint32_t sa_count; + caddr32_t sa_attributes; +} crypto_object_set_attribute_value32_t; + +typedef struct crypto_object_find_init32 { + uint32_t fi_return_value; + crypto_session_id_t fi_session; + uint32_t fi_count; + caddr32_t fi_attributes; +} crypto_object_find_init32_t; + +typedef struct crypto_object_find_update32 { + uint32_t fu_return_value; + crypto_session_id_t fu_session; + uint32_t fu_max_count; + uint32_t fu_count; + caddr32_t fu_handles; +} crypto_object_find_update32_t; + +typedef struct crypto_object_find_final32 { + uint32_t ff_return_value; + crypto_session_id_t ff_session; +} crypto_object_find_final32_t; + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_OBJECT_CREATE CRYPTO(100) +#define CRYPTO_OBJECT_COPY CRYPTO(101) +#define CRYPTO_OBJECT_DESTROY CRYPTO(102) +#define CRYPTO_OBJECT_GET_ATTRIBUTE_VALUE CRYPTO(103) +#define CRYPTO_OBJECT_GET_SIZE CRYPTO(104) +#define CRYPTO_OBJECT_SET_ATTRIBUTE_VALUE CRYPTO(105) +#define CRYPTO_OBJECT_FIND_INIT CRYPTO(106) +#define CRYPTO_OBJECT_FIND_UPDATE CRYPTO(107) +#define CRYPTO_OBJECT_FIND_FINAL CRYPTO(108) + +/* + * Key Generation Ioctls + */ +typedef struct crypto_object_generate_key { + uint_t gk_return_value; + crypto_session_id_t gk_session; + crypto_object_id_t gk_handle; + crypto_mechanism_t gk_mechanism; + uint_t gk_count; + caddr_t gk_attributes; +} crypto_object_generate_key_t; + +typedef struct crypto_object_generate_key_pair { + uint_t kp_return_value; + crypto_session_id_t kp_session; + crypto_object_id_t kp_public_handle; + crypto_object_id_t kp_private_handle; + uint_t kp_public_count; + uint_t kp_private_count; + caddr_t kp_public_attributes; + caddr_t kp_private_attributes; + crypto_mechanism_t kp_mechanism; +} crypto_object_generate_key_pair_t; + +typedef struct crypto_object_wrap_key { + uint_t wk_return_value; + crypto_session_id_t wk_session; + crypto_mechanism_t wk_mechanism; + crypto_key_t wk_wrapping_key; + crypto_object_id_t wk_object_handle; + size_t wk_wrapped_key_len; + caddr_t wk_wrapped_key; +} crypto_object_wrap_key_t; + +typedef struct crypto_object_unwrap_key { + uint_t uk_return_value; + crypto_session_id_t uk_session; + crypto_mechanism_t uk_mechanism; + crypto_key_t uk_unwrapping_key; + crypto_object_id_t uk_object_handle; + size_t uk_wrapped_key_len; + caddr_t uk_wrapped_key; + uint_t uk_count; + caddr_t uk_attributes; +} crypto_object_unwrap_key_t; + +typedef struct crypto_derive_key { + uint_t dk_return_value; + crypto_session_id_t dk_session; + crypto_mechanism_t dk_mechanism; + crypto_key_t dk_base_key; + crypto_object_id_t dk_object_handle; + uint_t dk_count; + caddr_t dk_attributes; +} crypto_derive_key_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_object_generate_key32 { + uint32_t gk_return_value; + crypto_session_id_t gk_session; + crypto_object_id_t gk_handle; + crypto_mechanism32_t gk_mechanism; + uint32_t gk_count; + caddr32_t gk_attributes; +} crypto_object_generate_key32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +typedef struct crypto_object_generate_key_pair32 { + uint32_t kp_return_value; + crypto_session_id_t kp_session; + crypto_object_id_t kp_public_handle; + crypto_object_id_t kp_private_handle; + uint32_t kp_public_count; + uint32_t kp_private_count; + caddr32_t kp_public_attributes; + caddr32_t kp_private_attributes; + crypto_mechanism32_t kp_mechanism; +} crypto_object_generate_key_pair32_t; + +typedef struct crypto_object_wrap_key32 { + uint32_t wk_return_value; + crypto_session_id_t wk_session; + crypto_mechanism32_t wk_mechanism; + crypto_key32_t wk_wrapping_key; + crypto_object_id_t wk_object_handle; + size32_t wk_wrapped_key_len; + caddr32_t wk_wrapped_key; +} crypto_object_wrap_key32_t; + +typedef struct crypto_object_unwrap_key32 { + uint32_t uk_return_value; + crypto_session_id_t uk_session; + crypto_mechanism32_t uk_mechanism; + crypto_key32_t uk_unwrapping_key; + crypto_object_id_t uk_object_handle; + size32_t uk_wrapped_key_len; + caddr32_t uk_wrapped_key; + uint32_t uk_count; + caddr32_t uk_attributes; +} crypto_object_unwrap_key32_t; + +typedef struct crypto_derive_key32 { + uint32_t dk_return_value; + crypto_session_id_t dk_session; + crypto_mechanism32_t dk_mechanism; + crypto_key32_t dk_base_key; + crypto_object_id_t dk_object_handle; + uint32_t dk_count; + caddr32_t dk_attributes; +} crypto_derive_key32_t; + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_GENERATE_KEY CRYPTO(110) +#define CRYPTO_GENERATE_KEY_PAIR CRYPTO(111) +#define CRYPTO_WRAP_KEY CRYPTO(112) +#define CRYPTO_UNWRAP_KEY CRYPTO(113) +#define CRYPTO_DERIVE_KEY CRYPTO(114) + +/* + * Provider Management Ioctls + */ + +typedef struct crypto_get_provider_list { + uint_t pl_return_value; + uint_t pl_count; + crypto_provider_entry_t pl_list[1]; +} crypto_get_provider_list_t; + +typedef struct crypto_provider_data { + uchar_t pd_prov_desc[CRYPTO_PROVIDER_DESCR_MAX_LEN]; + uchar_t pd_label[CRYPTO_EXT_SIZE_LABEL]; + uchar_t pd_manufacturerID[CRYPTO_EXT_SIZE_MANUF]; + uchar_t pd_model[CRYPTO_EXT_SIZE_MODEL]; + uchar_t pd_serial_number[CRYPTO_EXT_SIZE_SERIAL]; + ulong_t pd_flags; + ulong_t pd_max_session_count; + ulong_t pd_session_count; + ulong_t pd_max_rw_session_count; + ulong_t pd_rw_session_count; + ulong_t pd_max_pin_len; + ulong_t pd_min_pin_len; + ulong_t pd_total_public_memory; + ulong_t pd_free_public_memory; + ulong_t pd_total_private_memory; + ulong_t pd_free_private_memory; + crypto_version_t pd_hardware_version; + crypto_version_t pd_firmware_version; + uchar_t pd_time[CRYPTO_EXT_SIZE_TIME]; +} crypto_provider_data_t; + +typedef struct crypto_get_provider_info { + uint_t gi_return_value; + crypto_provider_id_t gi_provider_id; + crypto_provider_data_t gi_provider_data; +} crypto_get_provider_info_t; + +typedef struct crypto_get_provider_mechanisms { + uint_t pm_return_value; + crypto_provider_id_t pm_provider_id; + uint_t pm_count; + crypto_mech_name_t pm_list[1]; +} crypto_get_provider_mechanisms_t; + +typedef struct crypto_get_provider_mechanism_info { + uint_t mi_return_value; + crypto_provider_id_t mi_provider_id; + crypto_mech_name_t mi_mechanism_name; + uint32_t mi_min_key_size; + uint32_t mi_max_key_size; + uint32_t mi_flags; +} crypto_get_provider_mechanism_info_t; + +typedef struct crypto_init_token { + uint_t it_return_value; + crypto_provider_id_t it_provider_id; + caddr_t it_pin; + size_t it_pin_len; + caddr_t it_label; +} crypto_init_token_t; + +typedef struct crypto_init_pin { + uint_t ip_return_value; + crypto_session_id_t ip_session; + caddr_t ip_pin; + size_t ip_pin_len; +} crypto_init_pin_t; + +typedef struct crypto_set_pin { + uint_t sp_return_value; + crypto_session_id_t sp_session; + caddr_t sp_old_pin; + size_t sp_old_len; + caddr_t sp_new_pin; + size_t sp_new_len; +} crypto_set_pin_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +typedef struct crypto_get_provider_list32 { + uint32_t pl_return_value; + uint32_t pl_count; + crypto_provider_entry_t pl_list[1]; +} crypto_get_provider_list32_t; + +typedef struct crypto_version32 { + uchar_t cv_major; + uchar_t cv_minor; +} crypto_version32_t; + +typedef struct crypto_provider_data32 { + uchar_t pd_prov_desc[CRYPTO_PROVIDER_DESCR_MAX_LEN]; + uchar_t pd_label[CRYPTO_EXT_SIZE_LABEL]; + uchar_t pd_manufacturerID[CRYPTO_EXT_SIZE_MANUF]; + uchar_t pd_model[CRYPTO_EXT_SIZE_MODEL]; + uchar_t pd_serial_number[CRYPTO_EXT_SIZE_SERIAL]; + uint32_t pd_flags; + uint32_t pd_max_session_count; + uint32_t pd_session_count; + uint32_t pd_max_rw_session_count; + uint32_t pd_rw_session_count; + uint32_t pd_max_pin_len; + uint32_t pd_min_pin_len; + uint32_t pd_total_public_memory; + uint32_t pd_free_public_memory; + uint32_t pd_total_private_memory; + uint32_t pd_free_private_memory; + crypto_version32_t pd_hardware_version; + crypto_version32_t pd_firmware_version; + uchar_t pd_time[CRYPTO_EXT_SIZE_TIME]; +} crypto_provider_data32_t; + +typedef struct crypto_get_provider_info32 { + uint32_t gi_return_value; + crypto_provider_id_t gi_provider_id; + crypto_provider_data32_t gi_provider_data; +} crypto_get_provider_info32_t; + +typedef struct crypto_get_provider_mechanisms32 { + uint32_t pm_return_value; + crypto_provider_id_t pm_provider_id; + uint32_t pm_count; + crypto_mech_name_t pm_list[1]; +} crypto_get_provider_mechanisms32_t; + +typedef struct crypto_init_token32 { + uint32_t it_return_value; + crypto_provider_id_t it_provider_id; + caddr32_t it_pin; + size32_t it_pin_len; + caddr32_t it_label; +} crypto_init_token32_t; + +typedef struct crypto_init_pin32 { + uint32_t ip_return_value; + crypto_session_id_t ip_session; + caddr32_t ip_pin; + size32_t ip_pin_len; +} crypto_init_pin32_t; + +typedef struct crypto_set_pin32 { + uint32_t sp_return_value; + crypto_session_id_t sp_session; + caddr32_t sp_old_pin; + size32_t sp_old_len; + caddr32_t sp_new_pin; + size32_t sp_new_len; +} crypto_set_pin32_t; + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_GET_PROVIDER_LIST CRYPTO(120) +#define CRYPTO_GET_PROVIDER_INFO CRYPTO(121) +#define CRYPTO_GET_PROVIDER_MECHANISMS CRYPTO(122) +#define CRYPTO_GET_PROVIDER_MECHANISM_INFO CRYPTO(123) +#define CRYPTO_INIT_TOKEN CRYPTO(124) +#define CRYPTO_INIT_PIN CRYPTO(125) +#define CRYPTO_SET_PIN CRYPTO(126) + +/* + * No (Key) Store Key Generation Ioctls + */ +typedef struct crypto_nostore_generate_key { + uint_t ngk_return_value; + crypto_session_id_t ngk_session; + crypto_mechanism_t ngk_mechanism; + uint_t ngk_in_count; + uint_t ngk_out_count; + caddr_t ngk_in_attributes; + caddr_t ngk_out_attributes; +} crypto_nostore_generate_key_t; + +typedef struct crypto_nostore_generate_key_pair { + uint_t nkp_return_value; + crypto_session_id_t nkp_session; + uint_t nkp_in_public_count; + uint_t nkp_in_private_count; + uint_t nkp_out_public_count; + uint_t nkp_out_private_count; + caddr_t nkp_in_public_attributes; + caddr_t nkp_in_private_attributes; + caddr_t nkp_out_public_attributes; + caddr_t nkp_out_private_attributes; + crypto_mechanism_t nkp_mechanism; +} crypto_nostore_generate_key_pair_t; + +typedef struct crypto_nostore_derive_key { + uint_t ndk_return_value; + crypto_session_id_t ndk_session; + crypto_mechanism_t ndk_mechanism; + crypto_key_t ndk_base_key; + uint_t ndk_in_count; + uint_t ndk_out_count; + caddr_t ndk_in_attributes; + caddr_t ndk_out_attributes; +} crypto_nostore_derive_key_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +typedef struct crypto_nostore_generate_key32 { + uint32_t ngk_return_value; + crypto_session_id_t ngk_session; + crypto_mechanism32_t ngk_mechanism; + uint32_t ngk_in_count; + uint32_t ngk_out_count; + caddr32_t ngk_in_attributes; + caddr32_t ngk_out_attributes; +} crypto_nostore_generate_key32_t; + +typedef struct crypto_nostore_generate_key_pair32 { + uint32_t nkp_return_value; + crypto_session_id_t nkp_session; + uint32_t nkp_in_public_count; + uint32_t nkp_in_private_count; + uint32_t nkp_out_public_count; + uint32_t nkp_out_private_count; + caddr32_t nkp_in_public_attributes; + caddr32_t nkp_in_private_attributes; + caddr32_t nkp_out_public_attributes; + caddr32_t nkp_out_private_attributes; + crypto_mechanism32_t nkp_mechanism; +} crypto_nostore_generate_key_pair32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack(4) +#endif + +typedef struct crypto_nostore_derive_key32 { + uint32_t ndk_return_value; + crypto_session_id_t ndk_session; + crypto_mechanism32_t ndk_mechanism; + crypto_key32_t ndk_base_key; + uint32_t ndk_in_count; + uint32_t ndk_out_count; + caddr32_t ndk_in_attributes; + caddr32_t ndk_out_attributes; +} crypto_nostore_derive_key32_t; + +#if _LONG_LONG_ALIGNMENT == 8 && _LONG_LONG_ALIGNMENT_32 == 4 +#pragma pack() +#endif + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_NOSTORE_GENERATE_KEY CRYPTO(127) +#define CRYPTO_NOSTORE_GENERATE_KEY_PAIR CRYPTO(128) +#define CRYPTO_NOSTORE_DERIVE_KEY CRYPTO(129) + +/* + * Mechanism Ioctls + */ + +typedef struct crypto_get_mechanism_list { + uint_t ml_return_value; + uint_t ml_count; + crypto_mech_name_t ml_list[1]; +} crypto_get_mechanism_list_t; + +typedef struct crypto_get_all_mechanism_info { + uint_t mi_return_value; + crypto_mech_name_t mi_mechanism_name; + uint_t mi_count; + crypto_mechanism_info_t mi_list[1]; +} crypto_get_all_mechanism_info_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +typedef struct crypto_get_mechanism_list32 { + uint32_t ml_return_value; + uint32_t ml_count; + crypto_mech_name_t ml_list[1]; +} crypto_get_mechanism_list32_t; + +typedef struct crypto_get_all_mechanism_info32 { + uint32_t mi_return_value; + crypto_mech_name_t mi_mechanism_name; + uint32_t mi_count; + crypto_mechanism_info32_t mi_list[1]; +} crypto_get_all_mechanism_info32_t; + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_GET_MECHANISM_LIST CRYPTO(140) +#define CRYPTO_GET_ALL_MECHANISM_INFO CRYPTO(141) + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRYPTO_IOCTL_H */ diff --git a/module/icp/include/sys/crypto/ioctladmin.h b/module/icp/include/sys/crypto/ioctladmin.h new file mode 100644 index 000000000000..24babd7755cc --- /dev/null +++ b/module/icp/include/sys/crypto/ioctladmin.h @@ -0,0 +1,136 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CRYPTO_IOCTLADMIN_H +#define _SYS_CRYPTO_IOCTLADMIN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#define ADMIN_IOCTL_DEVICE "/dev/cryptoadm" + +#define CRYPTOADMIN(x) (('y' << 8) | (x)) + +/* + * Administrative IOCTLs + */ + +typedef struct crypto_get_dev_list { + uint_t dl_return_value; + uint_t dl_dev_count; + crypto_dev_list_entry_t dl_devs[1]; +} crypto_get_dev_list_t; + +typedef struct crypto_get_soft_list { + uint_t sl_return_value; + uint_t sl_soft_count; + size_t sl_soft_len; + caddr_t sl_soft_names; +} crypto_get_soft_list_t; + +typedef struct crypto_get_dev_info { + uint_t di_return_value; + char di_dev_name[MAXNAMELEN]; + uint_t di_dev_instance; + uint_t di_count; + crypto_mech_name_t di_list[1]; +} crypto_get_dev_info_t; + +typedef struct crypto_get_soft_info { + uint_t si_return_value; + char si_name[MAXNAMELEN]; + uint_t si_count; + crypto_mech_name_t si_list[1]; +} crypto_get_soft_info_t; + +typedef struct crypto_load_dev_disabled { + uint_t dd_return_value; + char dd_dev_name[MAXNAMELEN]; + uint_t dd_dev_instance; + uint_t dd_count; + crypto_mech_name_t dd_list[1]; +} crypto_load_dev_disabled_t; + +typedef struct crypto_load_soft_disabled { + uint_t sd_return_value; + char sd_name[MAXNAMELEN]; + uint_t sd_count; + crypto_mech_name_t sd_list[1]; +} crypto_load_soft_disabled_t; + +typedef struct crypto_unload_soft_module { + uint_t sm_return_value; + char sm_name[MAXNAMELEN]; +} crypto_unload_soft_module_t; + +typedef struct crypto_load_soft_config { + uint_t sc_return_value; + char sc_name[MAXNAMELEN]; + uint_t sc_count; + crypto_mech_name_t sc_list[1]; +} crypto_load_soft_config_t; + +typedef struct crypto_load_door { + uint_t ld_return_value; + uint_t ld_did; +} crypto_load_door_t; + +#ifdef _KERNEL +#ifdef _SYSCALL32 + +typedef struct crypto_get_soft_list32 { + uint32_t sl_return_value; + uint32_t sl_soft_count; + size32_t sl_soft_len; + caddr32_t sl_soft_names; +} crypto_get_soft_list32_t; + +#endif /* _SYSCALL32 */ +#endif /* _KERNEL */ + +#define CRYPTO_GET_VERSION CRYPTOADMIN(1) +#define CRYPTO_GET_DEV_LIST CRYPTOADMIN(2) +#define CRYPTO_GET_SOFT_LIST CRYPTOADMIN(3) +#define CRYPTO_GET_DEV_INFO CRYPTOADMIN(4) +#define CRYPTO_GET_SOFT_INFO CRYPTOADMIN(5) +#define CRYPTO_LOAD_DEV_DISABLED CRYPTOADMIN(8) +#define CRYPTO_LOAD_SOFT_DISABLED CRYPTOADMIN(9) +#define CRYPTO_UNLOAD_SOFT_MODULE CRYPTOADMIN(10) +#define CRYPTO_LOAD_SOFT_CONFIG CRYPTOADMIN(11) +#define CRYPTO_POOL_CREATE CRYPTOADMIN(12) +#define CRYPTO_POOL_WAIT CRYPTOADMIN(13) +#define CRYPTO_POOL_RUN CRYPTOADMIN(14) +#define CRYPTO_LOAD_DOOR CRYPTOADMIN(15) + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRYPTO_IOCTLADMIN_H */ diff --git a/module/icp/include/sys/crypto/ops_impl.h b/module/icp/include/sys/crypto/ops_impl.h new file mode 100644 index 000000000000..230d74b063fc --- /dev/null +++ b/module/icp/include/sys/crypto/ops_impl.h @@ -0,0 +1,630 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CRYPTO_OPS_IMPL_H +#define _SYS_CRYPTO_OPS_IMPL_H + +/* + * Scheduler internal structures. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include + +/* + * The parameters needed for each function group are batched + * in one structure. This is much simpler than having a + * separate structure for each function. + * + * In some cases, a field is generically named to keep the + * structure small. The comments indicate these cases. + */ +typedef struct kcf_digest_ops_params { + crypto_session_id_t do_sid; + crypto_mech_type_t do_framework_mechtype; + crypto_mechanism_t do_mech; + crypto_data_t *do_data; + crypto_data_t *do_digest; + crypto_key_t *do_digest_key; /* Argument for digest_key() */ +} kcf_digest_ops_params_t; + +typedef struct kcf_mac_ops_params { + crypto_session_id_t mo_sid; + crypto_mech_type_t mo_framework_mechtype; + crypto_mechanism_t mo_mech; + crypto_key_t *mo_key; + crypto_data_t *mo_data; + crypto_data_t *mo_mac; + crypto_spi_ctx_template_t mo_templ; +} kcf_mac_ops_params_t; + +typedef struct kcf_encrypt_ops_params { + crypto_session_id_t eo_sid; + crypto_mech_type_t eo_framework_mechtype; + crypto_mechanism_t eo_mech; + crypto_key_t *eo_key; + crypto_data_t *eo_plaintext; + crypto_data_t *eo_ciphertext; + crypto_spi_ctx_template_t eo_templ; +} kcf_encrypt_ops_params_t; + +typedef struct kcf_decrypt_ops_params { + crypto_session_id_t dop_sid; + crypto_mech_type_t dop_framework_mechtype; + crypto_mechanism_t dop_mech; + crypto_key_t *dop_key; + crypto_data_t *dop_ciphertext; + crypto_data_t *dop_plaintext; + crypto_spi_ctx_template_t dop_templ; +} kcf_decrypt_ops_params_t; + +typedef struct kcf_sign_ops_params { + crypto_session_id_t so_sid; + crypto_mech_type_t so_framework_mechtype; + crypto_mechanism_t so_mech; + crypto_key_t *so_key; + crypto_data_t *so_data; + crypto_data_t *so_signature; + crypto_spi_ctx_template_t so_templ; +} kcf_sign_ops_params_t; + +typedef struct kcf_verify_ops_params { + crypto_session_id_t vo_sid; + crypto_mech_type_t vo_framework_mechtype; + crypto_mechanism_t vo_mech; + crypto_key_t *vo_key; + crypto_data_t *vo_data; + crypto_data_t *vo_signature; + crypto_spi_ctx_template_t vo_templ; +} kcf_verify_ops_params_t; + +typedef struct kcf_encrypt_mac_ops_params { + crypto_session_id_t em_sid; + crypto_mech_type_t em_framework_encr_mechtype; + crypto_mechanism_t em_encr_mech; + crypto_key_t *em_encr_key; + crypto_mech_type_t em_framework_mac_mechtype; + crypto_mechanism_t em_mac_mech; + crypto_key_t *em_mac_key; + crypto_data_t *em_plaintext; + crypto_dual_data_t *em_ciphertext; + crypto_data_t *em_mac; + crypto_spi_ctx_template_t em_encr_templ; + crypto_spi_ctx_template_t em_mac_templ; +} kcf_encrypt_mac_ops_params_t; + +typedef struct kcf_mac_decrypt_ops_params { + crypto_session_id_t md_sid; + crypto_mech_type_t md_framework_mac_mechtype; + crypto_mechanism_t md_mac_mech; + crypto_key_t *md_mac_key; + crypto_mech_type_t md_framework_decr_mechtype; + crypto_mechanism_t md_decr_mech; + crypto_key_t *md_decr_key; + crypto_dual_data_t *md_ciphertext; + crypto_data_t *md_mac; + crypto_data_t *md_plaintext; + crypto_spi_ctx_template_t md_mac_templ; + crypto_spi_ctx_template_t md_decr_templ; +} kcf_mac_decrypt_ops_params_t; + +typedef struct kcf_random_number_ops_params { + crypto_session_id_t rn_sid; + uchar_t *rn_buf; + size_t rn_buflen; + uint_t rn_entropy_est; + uint32_t rn_flags; +} kcf_random_number_ops_params_t; + +/* + * so_pd is useful when the provider descriptor (pd) supplying the + * provider handle is different from the pd supplying the ops vector. + * This is the case for session open/close where so_pd can be the pd + * of a logical provider. The pd supplying the ops vector is passed + * as an argument to kcf_submit_request(). + */ +typedef struct kcf_session_ops_params { + crypto_session_id_t *so_sid_ptr; + crypto_session_id_t so_sid; + crypto_user_type_t so_user_type; + char *so_pin; + size_t so_pin_len; + kcf_provider_desc_t *so_pd; +} kcf_session_ops_params_t; + +typedef struct kcf_object_ops_params { + crypto_session_id_t oo_sid; + crypto_object_id_t oo_object_id; + crypto_object_attribute_t *oo_template; + uint_t oo_attribute_count; + crypto_object_id_t *oo_object_id_ptr; + size_t *oo_object_size; + void **oo_find_init_pp_ptr; + void *oo_find_pp; + uint_t oo_max_object_count; + uint_t *oo_object_count_ptr; +} kcf_object_ops_params_t; + +/* + * ko_key is used to encode wrapping key in key_wrap() and + * unwrapping key in key_unwrap(). ko_key_template and + * ko_key_attribute_count are used to encode public template + * and public template attr count in key_generate_pair(). + * kops->ko_key_object_id_ptr is used to encode public key + * in key_generate_pair(). + */ +typedef struct kcf_key_ops_params { + crypto_session_id_t ko_sid; + crypto_mech_type_t ko_framework_mechtype; + crypto_mechanism_t ko_mech; + crypto_object_attribute_t *ko_key_template; + uint_t ko_key_attribute_count; + crypto_object_id_t *ko_key_object_id_ptr; + crypto_object_attribute_t *ko_private_key_template; + uint_t ko_private_key_attribute_count; + crypto_object_id_t *ko_private_key_object_id_ptr; + crypto_key_t *ko_key; + uchar_t *ko_wrapped_key; + size_t *ko_wrapped_key_len_ptr; + crypto_object_attribute_t *ko_out_template1; + crypto_object_attribute_t *ko_out_template2; + uint_t ko_out_attribute_count1; + uint_t ko_out_attribute_count2; +} kcf_key_ops_params_t; + +/* + * po_pin and po_pin_len are used to encode new_pin and new_pin_len + * when wrapping set_pin() function parameters. + * + * po_pd is useful when the provider descriptor (pd) supplying the + * provider handle is different from the pd supplying the ops vector. + * This is true for the ext_info provider entry point where po_pd + * can be the pd of a logical provider. The pd supplying the ops vector + * is passed as an argument to kcf_submit_request(). + */ +typedef struct kcf_provmgmt_ops_params { + crypto_session_id_t po_sid; + char *po_pin; + size_t po_pin_len; + char *po_old_pin; + size_t po_old_pin_len; + char *po_label; + crypto_provider_ext_info_t *po_ext_info; + kcf_provider_desc_t *po_pd; +} kcf_provmgmt_ops_params_t; + +/* + * The operation type within a function group. + */ +typedef enum kcf_op_type { + /* common ops for all mechanisms */ + KCF_OP_INIT = 1, + KCF_OP_SINGLE, /* pkcs11 sense. So, INIT is already done */ + KCF_OP_UPDATE, + KCF_OP_FINAL, + KCF_OP_ATOMIC, + + /* digest_key op */ + KCF_OP_DIGEST_KEY, + + /* mac specific op */ + KCF_OP_MAC_VERIFY_ATOMIC, + + /* mac/cipher specific op */ + KCF_OP_MAC_VERIFY_DECRYPT_ATOMIC, + + /* sign_recover ops */ + KCF_OP_SIGN_RECOVER_INIT, + KCF_OP_SIGN_RECOVER, + KCF_OP_SIGN_RECOVER_ATOMIC, + + /* verify_recover ops */ + KCF_OP_VERIFY_RECOVER_INIT, + KCF_OP_VERIFY_RECOVER, + KCF_OP_VERIFY_RECOVER_ATOMIC, + + /* random number ops */ + KCF_OP_RANDOM_SEED, + KCF_OP_RANDOM_GENERATE, + + /* session management ops */ + KCF_OP_SESSION_OPEN, + KCF_OP_SESSION_CLOSE, + KCF_OP_SESSION_LOGIN, + KCF_OP_SESSION_LOGOUT, + + /* object management ops */ + KCF_OP_OBJECT_CREATE, + KCF_OP_OBJECT_COPY, + KCF_OP_OBJECT_DESTROY, + KCF_OP_OBJECT_GET_SIZE, + KCF_OP_OBJECT_GET_ATTRIBUTE_VALUE, + KCF_OP_OBJECT_SET_ATTRIBUTE_VALUE, + KCF_OP_OBJECT_FIND_INIT, + KCF_OP_OBJECT_FIND, + KCF_OP_OBJECT_FIND_FINAL, + + /* key management ops */ + KCF_OP_KEY_GENERATE, + KCF_OP_KEY_GENERATE_PAIR, + KCF_OP_KEY_WRAP, + KCF_OP_KEY_UNWRAP, + KCF_OP_KEY_DERIVE, + KCF_OP_KEY_CHECK, + + /* provider management ops */ + KCF_OP_MGMT_EXTINFO, + KCF_OP_MGMT_INITTOKEN, + KCF_OP_MGMT_INITPIN, + KCF_OP_MGMT_SETPIN +} kcf_op_type_t; + +/* + * The operation groups that need wrapping of parameters. This is somewhat + * similar to the function group type in spi.h except that this also includes + * all the functions that don't have a mechanism. + * + * The wrapper macros should never take these enum values as an argument. + * Rather, they are assigned in the macro itself since they are known + * from the macro name. + */ +typedef enum kcf_op_group { + KCF_OG_DIGEST = 1, + KCF_OG_MAC, + KCF_OG_ENCRYPT, + KCF_OG_DECRYPT, + KCF_OG_SIGN, + KCF_OG_VERIFY, + KCF_OG_ENCRYPT_MAC, + KCF_OG_MAC_DECRYPT, + KCF_OG_RANDOM, + KCF_OG_SESSION, + KCF_OG_OBJECT, + KCF_OG_KEY, + KCF_OG_PROVMGMT, + KCF_OG_NOSTORE_KEY +} kcf_op_group_t; + +/* + * The kcf_op_type_t enum values used here should be only for those + * operations for which there is a k-api routine in sys/crypto/api.h. + */ +#define IS_INIT_OP(ftype) ((ftype) == KCF_OP_INIT) +#define IS_SINGLE_OP(ftype) ((ftype) == KCF_OP_SINGLE) +#define IS_UPDATE_OP(ftype) ((ftype) == KCF_OP_UPDATE) +#define IS_FINAL_OP(ftype) ((ftype) == KCF_OP_FINAL) +#define IS_ATOMIC_OP(ftype) ( \ + (ftype) == KCF_OP_ATOMIC || (ftype) == KCF_OP_MAC_VERIFY_ATOMIC || \ + (ftype) == KCF_OP_MAC_VERIFY_DECRYPT_ATOMIC || \ + (ftype) == KCF_OP_SIGN_RECOVER_ATOMIC || \ + (ftype) == KCF_OP_VERIFY_RECOVER_ATOMIC) + +/* + * Keep the parameters associated with a request around. + * We need to pass them to the SPI. + */ +typedef struct kcf_req_params { + kcf_op_group_t rp_opgrp; + kcf_op_type_t rp_optype; + + union { + kcf_digest_ops_params_t digest_params; + kcf_mac_ops_params_t mac_params; + kcf_encrypt_ops_params_t encrypt_params; + kcf_decrypt_ops_params_t decrypt_params; + kcf_sign_ops_params_t sign_params; + kcf_verify_ops_params_t verify_params; + kcf_encrypt_mac_ops_params_t encrypt_mac_params; + kcf_mac_decrypt_ops_params_t mac_decrypt_params; + kcf_random_number_ops_params_t random_number_params; + kcf_session_ops_params_t session_params; + kcf_object_ops_params_t object_params; + kcf_key_ops_params_t key_params; + kcf_provmgmt_ops_params_t provmgmt_params; + } rp_u; +} kcf_req_params_t; + + +/* + * The ioctl/k-api code should bundle the parameters into a kcf_req_params_t + * structure before calling a scheduler routine. The following macros are + * available for that purpose. + * + * For the most part, the macro arguments closely correspond to the + * function parameters. In some cases, we use generic names. The comments + * for the structure should indicate these cases. + */ +#define KCF_WRAP_DIGEST_OPS_PARAMS(req, ftype, _sid, _mech, _key, \ + _data, _digest) { \ + kcf_digest_ops_params_t *dops = &(req)->rp_u.digest_params; \ + crypto_mechanism_t *mechp = _mech; \ + \ + (req)->rp_opgrp = KCF_OG_DIGEST; \ + (req)->rp_optype = ftype; \ + dops->do_sid = _sid; \ + if (mechp != NULL) { \ + dops->do_mech = *mechp; \ + dops->do_framework_mechtype = mechp->cm_type; \ + } \ + dops->do_digest_key = _key; \ + dops->do_data = _data; \ + dops->do_digest = _digest; \ +} + +#define KCF_WRAP_MAC_OPS_PARAMS(req, ftype, _sid, _mech, _key, \ + _data, _mac, _templ) { \ + kcf_mac_ops_params_t *mops = &(req)->rp_u.mac_params; \ + crypto_mechanism_t *mechp = _mech; \ + \ + (req)->rp_opgrp = KCF_OG_MAC; \ + (req)->rp_optype = ftype; \ + mops->mo_sid = _sid; \ + if (mechp != NULL) { \ + mops->mo_mech = *mechp; \ + mops->mo_framework_mechtype = mechp->cm_type; \ + } \ + mops->mo_key = _key; \ + mops->mo_data = _data; \ + mops->mo_mac = _mac; \ + mops->mo_templ = _templ; \ +} + +#define KCF_WRAP_ENCRYPT_OPS_PARAMS(req, ftype, _sid, _mech, _key, \ + _plaintext, _ciphertext, _templ) { \ + kcf_encrypt_ops_params_t *cops = &(req)->rp_u.encrypt_params; \ + crypto_mechanism_t *mechp = _mech; \ + \ + (req)->rp_opgrp = KCF_OG_ENCRYPT; \ + (req)->rp_optype = ftype; \ + cops->eo_sid = _sid; \ + if (mechp != NULL) { \ + cops->eo_mech = *mechp; \ + cops->eo_framework_mechtype = mechp->cm_type; \ + } \ + cops->eo_key = _key; \ + cops->eo_plaintext = _plaintext; \ + cops->eo_ciphertext = _ciphertext; \ + cops->eo_templ = _templ; \ +} + +#define KCF_WRAP_DECRYPT_OPS_PARAMS(req, ftype, _sid, _mech, _key, \ + _ciphertext, _plaintext, _templ) { \ + kcf_decrypt_ops_params_t *cops = &(req)->rp_u.decrypt_params; \ + crypto_mechanism_t *mechp = _mech; \ + \ + (req)->rp_opgrp = KCF_OG_DECRYPT; \ + (req)->rp_optype = ftype; \ + cops->dop_sid = _sid; \ + if (mechp != NULL) { \ + cops->dop_mech = *mechp; \ + cops->dop_framework_mechtype = mechp->cm_type; \ + } \ + cops->dop_key = _key; \ + cops->dop_ciphertext = _ciphertext; \ + cops->dop_plaintext = _plaintext; \ + cops->dop_templ = _templ; \ +} + +#define KCF_WRAP_SIGN_OPS_PARAMS(req, ftype, _sid, _mech, _key, \ + _data, _signature, _templ) { \ + kcf_sign_ops_params_t *sops = &(req)->rp_u.sign_params; \ + crypto_mechanism_t *mechp = _mech; \ + \ + (req)->rp_opgrp = KCF_OG_SIGN; \ + (req)->rp_optype = ftype; \ + sops->so_sid = _sid; \ + if (mechp != NULL) { \ + sops->so_mech = *mechp; \ + sops->so_framework_mechtype = mechp->cm_type; \ + } \ + sops->so_key = _key; \ + sops->so_data = _data; \ + sops->so_signature = _signature; \ + sops->so_templ = _templ; \ +} + +#define KCF_WRAP_VERIFY_OPS_PARAMS(req, ftype, _sid, _mech, _key, \ + _data, _signature, _templ) { \ + kcf_verify_ops_params_t *vops = &(req)->rp_u.verify_params; \ + crypto_mechanism_t *mechp = _mech; \ + \ + (req)->rp_opgrp = KCF_OG_VERIFY; \ + (req)->rp_optype = ftype; \ + vops->vo_sid = _sid; \ + if (mechp != NULL) { \ + vops->vo_mech = *mechp; \ + vops->vo_framework_mechtype = mechp->cm_type; \ + } \ + vops->vo_key = _key; \ + vops->vo_data = _data; \ + vops->vo_signature = _signature; \ + vops->vo_templ = _templ; \ +} + +#define KCF_WRAP_ENCRYPT_MAC_OPS_PARAMS(req, ftype, _sid, _encr_key, \ + _mac_key, _plaintext, _ciphertext, _mac, _encr_templ, _mac_templ) { \ + kcf_encrypt_mac_ops_params_t *cmops = &(req)->rp_u.encrypt_mac_params; \ + \ + (req)->rp_opgrp = KCF_OG_ENCRYPT_MAC; \ + (req)->rp_optype = ftype; \ + cmops->em_sid = _sid; \ + cmops->em_encr_key = _encr_key; \ + cmops->em_mac_key = _mac_key; \ + cmops->em_plaintext = _plaintext; \ + cmops->em_ciphertext = _ciphertext; \ + cmops->em_mac = _mac; \ + cmops->em_encr_templ = _encr_templ; \ + cmops->em_mac_templ = _mac_templ; \ +} + +#define KCF_WRAP_MAC_DECRYPT_OPS_PARAMS(req, ftype, _sid, _mac_key, \ + _decr_key, _ciphertext, _mac, _plaintext, _mac_templ, _decr_templ) { \ + kcf_mac_decrypt_ops_params_t *cmops = &(req)->rp_u.mac_decrypt_params; \ + \ + (req)->rp_opgrp = KCF_OG_MAC_DECRYPT; \ + (req)->rp_optype = ftype; \ + cmops->md_sid = _sid; \ + cmops->md_mac_key = _mac_key; \ + cmops->md_decr_key = _decr_key; \ + cmops->md_ciphertext = _ciphertext; \ + cmops->md_mac = _mac; \ + cmops->md_plaintext = _plaintext; \ + cmops->md_mac_templ = _mac_templ; \ + cmops->md_decr_templ = _decr_templ; \ +} + +#define KCF_WRAP_RANDOM_OPS_PARAMS(req, ftype, _sid, _buf, _buflen, \ + _est, _flags) { \ + kcf_random_number_ops_params_t *rops = \ + &(req)->rp_u.random_number_params; \ + \ + (req)->rp_opgrp = KCF_OG_RANDOM; \ + (req)->rp_optype = ftype; \ + rops->rn_sid = _sid; \ + rops->rn_buf = _buf; \ + rops->rn_buflen = _buflen; \ + rops->rn_entropy_est = _est; \ + rops->rn_flags = _flags; \ +} + +#define KCF_WRAP_SESSION_OPS_PARAMS(req, ftype, _sid_ptr, _sid, \ + _user_type, _pin, _pin_len, _pd) { \ + kcf_session_ops_params_t *sops = &(req)->rp_u.session_params; \ + \ + (req)->rp_opgrp = KCF_OG_SESSION; \ + (req)->rp_optype = ftype; \ + sops->so_sid_ptr = _sid_ptr; \ + sops->so_sid = _sid; \ + sops->so_user_type = _user_type; \ + sops->so_pin = _pin; \ + sops->so_pin_len = _pin_len; \ + sops->so_pd = _pd; \ +} + +#define KCF_WRAP_OBJECT_OPS_PARAMS(req, ftype, _sid, _object_id, \ + _template, _attribute_count, _object_id_ptr, _object_size, \ + _find_init_pp_ptr, _find_pp, _max_object_count, _object_count_ptr) { \ + kcf_object_ops_params_t *jops = &(req)->rp_u.object_params; \ + \ + (req)->rp_opgrp = KCF_OG_OBJECT; \ + (req)->rp_optype = ftype; \ + jops->oo_sid = _sid; \ + jops->oo_object_id = _object_id; \ + jops->oo_template = _template; \ + jops->oo_attribute_count = _attribute_count; \ + jops->oo_object_id_ptr = _object_id_ptr; \ + jops->oo_object_size = _object_size; \ + jops->oo_find_init_pp_ptr = _find_init_pp_ptr; \ + jops->oo_find_pp = _find_pp; \ + jops->oo_max_object_count = _max_object_count; \ + jops->oo_object_count_ptr = _object_count_ptr; \ +} + +#define KCF_WRAP_KEY_OPS_PARAMS(req, ftype, _sid, _mech, _key_template, \ + _key_attribute_count, _key_object_id_ptr, _private_key_template, \ + _private_key_attribute_count, _private_key_object_id_ptr, \ + _key, _wrapped_key, _wrapped_key_len_ptr) { \ + kcf_key_ops_params_t *kops = &(req)->rp_u.key_params; \ + crypto_mechanism_t *mechp = _mech; \ + \ + (req)->rp_opgrp = KCF_OG_KEY; \ + (req)->rp_optype = ftype; \ + kops->ko_sid = _sid; \ + if (mechp != NULL) { \ + kops->ko_mech = *mechp; \ + kops->ko_framework_mechtype = mechp->cm_type; \ + } \ + kops->ko_key_template = _key_template; \ + kops->ko_key_attribute_count = _key_attribute_count; \ + kops->ko_key_object_id_ptr = _key_object_id_ptr; \ + kops->ko_private_key_template = _private_key_template; \ + kops->ko_private_key_attribute_count = _private_key_attribute_count; \ + kops->ko_private_key_object_id_ptr = _private_key_object_id_ptr; \ + kops->ko_key = _key; \ + kops->ko_wrapped_key = _wrapped_key; \ + kops->ko_wrapped_key_len_ptr = _wrapped_key_len_ptr; \ +} + +#define KCF_WRAP_PROVMGMT_OPS_PARAMS(req, ftype, _sid, _old_pin, \ + _old_pin_len, _pin, _pin_len, _label, _ext_info, _pd) { \ + kcf_provmgmt_ops_params_t *pops = &(req)->rp_u.provmgmt_params; \ + \ + (req)->rp_opgrp = KCF_OG_PROVMGMT; \ + (req)->rp_optype = ftype; \ + pops->po_sid = _sid; \ + pops->po_pin = _pin; \ + pops->po_pin_len = _pin_len; \ + pops->po_old_pin = _old_pin; \ + pops->po_old_pin_len = _old_pin_len; \ + pops->po_label = _label; \ + pops->po_ext_info = _ext_info; \ + pops->po_pd = _pd; \ +} + +#define KCF_WRAP_NOSTORE_KEY_OPS_PARAMS(req, ftype, _sid, _mech, \ + _key_template, _key_attribute_count, _private_key_template, \ + _private_key_attribute_count, _key, _out_template1, \ + _out_attribute_count1, _out_template2, _out_attribute_count2) { \ + kcf_key_ops_params_t *kops = &(req)->rp_u.key_params; \ + crypto_mechanism_t *mechp = _mech; \ + \ + (req)->rp_opgrp = KCF_OG_NOSTORE_KEY; \ + (req)->rp_optype = ftype; \ + kops->ko_sid = _sid; \ + if (mechp != NULL) { \ + kops->ko_mech = *mechp; \ + kops->ko_framework_mechtype = mechp->cm_type; \ + } \ + kops->ko_key_template = _key_template; \ + kops->ko_key_attribute_count = _key_attribute_count; \ + kops->ko_key_object_id_ptr = NULL; \ + kops->ko_private_key_template = _private_key_template; \ + kops->ko_private_key_attribute_count = _private_key_attribute_count; \ + kops->ko_private_key_object_id_ptr = NULL; \ + kops->ko_key = _key; \ + kops->ko_wrapped_key = NULL; \ + kops->ko_wrapped_key_len_ptr = 0; \ + kops->ko_out_template1 = _out_template1; \ + kops->ko_out_template2 = _out_template2; \ + kops->ko_out_attribute_count1 = _out_attribute_count1; \ + kops->ko_out_attribute_count2 = _out_attribute_count2; \ +} + +#define KCF_SET_PROVIDER_MECHNUM(fmtype, pd, mechp) \ + (mechp)->cm_type = \ + KCF_TO_PROV_MECHNUM(pd, fmtype); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRYPTO_OPS_IMPL_H */ diff --git a/module/icp/include/sys/crypto/sched_impl.h b/module/icp/include/sys/crypto/sched_impl.h new file mode 100644 index 000000000000..32ffa774957b --- /dev/null +++ b/module/icp/include/sys/crypto/sched_impl.h @@ -0,0 +1,531 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CRYPTO_SCHED_IMPL_H +#define _SYS_CRYPTO_SCHED_IMPL_H + +/* + * Scheduler internal structures. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include +#include +#include +#include + +typedef void (kcf_func_t)(void *, int); + +typedef enum kcf_req_status { + REQ_ALLOCATED = 1, + REQ_WAITING, /* At the framework level */ + REQ_INPROGRESS, /* At the provider level */ + REQ_DONE, + REQ_CANCELED +} kcf_req_status_t; + +typedef enum kcf_call_type { + CRYPTO_SYNCH = 1, + CRYPTO_ASYNCH +} kcf_call_type_t; + +#define CHECK_RESTRICT(crq) (crq != NULL && \ + ((crq)->cr_flag & CRYPTO_RESTRICTED)) + +#define CHECK_RESTRICT_FALSE B_FALSE + +#define CHECK_FASTPATH(crq, pd) ((crq) == NULL || \ + !((crq)->cr_flag & CRYPTO_ALWAYS_QUEUE)) && \ + (pd)->pd_prov_type == CRYPTO_SW_PROVIDER + +#define KCF_KMFLAG(crq) (((crq) == NULL) ? KM_SLEEP : KM_NOSLEEP) + +/* + * The framework keeps an internal handle to use in the adaptive + * asynchronous case. This is the case when a client has the + * CRYPTO_ALWAYS_QUEUE bit clear and a software provider is used for + * the request. The request is completed in the context of the calling + * thread and kernel memory must be allocated with KM_NOSLEEP. + * + * The framework passes a pointer to the handle in crypto_req_handle_t + * argument when it calls the SPI of the software provider. The macros + * KCF_RHNDL() and KCF_SWFP_RHNDL() are used to do this. + * + * When a provider asks the framework for kmflag value via + * crypto_kmflag(9S) we use REQHNDL2_KMFLAG() macro. + */ +extern ulong_t kcf_swprov_hndl; +#define KCF_RHNDL(kmflag) (((kmflag) == KM_SLEEP) ? NULL : &kcf_swprov_hndl) +#define KCF_SWFP_RHNDL(crq) (((crq) == NULL) ? NULL : &kcf_swprov_hndl) +#define REQHNDL2_KMFLAG(rhndl) \ + ((rhndl == &kcf_swprov_hndl) ? KM_NOSLEEP : KM_SLEEP) + +/* Internal call_req flags. They start after the public ones in api.h */ + +#define CRYPTO_SETDUAL 0x00001000 /* Set the 'cont' boolean before */ + /* submitting the request */ +#define KCF_ISDUALREQ(crq) \ + (((crq) == NULL) ? B_FALSE : (crq->cr_flag & CRYPTO_SETDUAL)) + +typedef struct kcf_prov_tried { + kcf_provider_desc_t *pt_pd; + struct kcf_prov_tried *pt_next; +} kcf_prov_tried_t; + +#define IS_FG_SUPPORTED(mdesc, fg) \ + (((mdesc)->pm_mech_info.cm_func_group_mask & (fg)) != 0) + +#define IS_PROVIDER_TRIED(pd, tlist) \ + (tlist != NULL && is_in_triedlist(pd, tlist)) + +#define IS_RECOVERABLE(error) \ + (error == CRYPTO_BUFFER_TOO_BIG || \ + error == CRYPTO_BUSY || \ + error == CRYPTO_DEVICE_ERROR || \ + error == CRYPTO_DEVICE_MEMORY || \ + error == CRYPTO_KEY_SIZE_RANGE || \ + error == CRYPTO_NO_PERMISSION) + +#define KCF_ATOMIC_INCR(x) atomic_add_32(&(x), 1) +#define KCF_ATOMIC_DECR(x) atomic_add_32(&(x), -1) + +/* + * Node structure for synchronous requests. + */ +typedef struct kcf_sreq_node { + /* Should always be the first field in this structure */ + kcf_call_type_t sn_type; + /* + * sn_cv and sr_lock are used to wait for the + * operation to complete. sn_lock also protects + * the sn_state field. + */ + kcondvar_t sn_cv; + kmutex_t sn_lock; + kcf_req_status_t sn_state; + + /* + * Return value from the operation. This will be + * one of the CRYPTO_* errors defined in common.h. + */ + int sn_rv; + + /* + * parameters to call the SPI with. This can be + * a pointer as we know the caller context/stack stays. + */ + struct kcf_req_params *sn_params; + + /* Internal context for this request */ + struct kcf_context *sn_context; + + /* Provider handling this request */ + kcf_provider_desc_t *sn_provider; +} kcf_sreq_node_t; + +/* + * Node structure for asynchronous requests. A node can be on + * on a chain of requests hanging of the internal context + * structure and can be in the global software provider queue. + */ +typedef struct kcf_areq_node { + /* Should always be the first field in this structure */ + kcf_call_type_t an_type; + + /* an_lock protects the field an_state */ + kmutex_t an_lock; + kcf_req_status_t an_state; + crypto_call_req_t an_reqarg; + + /* + * parameters to call the SPI with. We need to + * save the params since the caller stack can go away. + */ + struct kcf_req_params an_params; + + /* + * The next two fields should be NULL for operations that + * don't need a context. + */ + /* Internal context for this request */ + struct kcf_context *an_context; + + /* next in chain of requests for context */ + struct kcf_areq_node *an_ctxchain_next; + + kcondvar_t an_turn_cv; + boolean_t an_is_my_turn; + boolean_t an_isdual; /* for internal reuse */ + + /* + * Next and previous nodes in the global software + * queue. These fields are NULL for a hardware + * provider since we use a taskq there. + */ + struct kcf_areq_node *an_next; + struct kcf_areq_node *an_prev; + + /* Provider handling this request */ + kcf_provider_desc_t *an_provider; + kcf_prov_tried_t *an_tried_plist; + + struct kcf_areq_node *an_idnext; /* Next in ID hash */ + struct kcf_areq_node *an_idprev; /* Prev in ID hash */ + kcondvar_t an_done; /* Signal request completion */ + uint_t an_refcnt; +} kcf_areq_node_t; + +#define KCF_AREQ_REFHOLD(areq) { \ + atomic_add_32(&(areq)->an_refcnt, 1); \ + ASSERT((areq)->an_refcnt != 0); \ +} + +#define KCF_AREQ_REFRELE(areq) { \ + ASSERT((areq)->an_refcnt != 0); \ + membar_exit(); \ + if (atomic_add_32_nv(&(areq)->an_refcnt, -1) == 0) \ + kcf_free_req(areq); \ +} + +#define GET_REQ_TYPE(arg) *((kcf_call_type_t *)(arg)) + +#define NOTIFY_CLIENT(areq, err) (*(areq)->an_reqarg.cr_callback_func)(\ + (areq)->an_reqarg.cr_callback_arg, err); + +/* For internally generated call requests for dual operations */ +typedef struct kcf_call_req { + crypto_call_req_t kr_callreq; /* external client call req */ + kcf_req_params_t kr_params; /* Params saved for next call */ + kcf_areq_node_t *kr_areq; /* Use this areq */ + off_t kr_saveoffset; + size_t kr_savelen; +} kcf_dual_req_t; + +/* + * The following are some what similar to macros in callo.h, which implement + * callout tables. + * + * The lower four bits of the ID are used to encode the table ID to + * index in to. The REQID_COUNTER_HIGH bit is used to avoid any check for + * wrap around when generating ID. We assume that there won't be a request + * which takes more time than 2^^(sizeof (long) - 5) other requests submitted + * after it. This ensures there won't be any ID collision. + */ +#define REQID_COUNTER_HIGH (1UL << (8 * sizeof (long) - 1)) +#define REQID_COUNTER_SHIFT 4 +#define REQID_COUNTER_LOW (1 << REQID_COUNTER_SHIFT) +#define REQID_TABLES 16 +#define REQID_TABLE_MASK (REQID_TABLES - 1) + +#define REQID_BUCKETS 512 +#define REQID_BUCKET_MASK (REQID_BUCKETS - 1) +#define REQID_HASH(id) (((id) >> REQID_COUNTER_SHIFT) & REQID_BUCKET_MASK) + +#define GET_REQID(areq) (areq)->an_reqarg.cr_reqid +#define SET_REQID(areq, val) GET_REQID(areq) = val + +/* + * Hash table for async requests. + */ +typedef struct kcf_reqid_table { + kmutex_t rt_lock; + crypto_req_id_t rt_curid; + kcf_areq_node_t *rt_idhash[REQID_BUCKETS]; +} kcf_reqid_table_t; + +/* + * Global software provider queue structure. Requests to be + * handled by a SW provider and have the ALWAYS_QUEUE flag set + * get queued here. + */ +typedef struct kcf_global_swq { + /* + * gs_cv and gs_lock are used to wait for new requests. + * gs_lock protects the changes to the queue. + */ + kcondvar_t gs_cv; + kmutex_t gs_lock; + uint_t gs_njobs; + uint_t gs_maxjobs; + kcf_areq_node_t *gs_first; + kcf_areq_node_t *gs_last; +} kcf_global_swq_t; + + +/* + * Internal representation of a canonical context. We contain crypto_ctx_t + * structure in order to have just one memory allocation. The SPI + * ((crypto_ctx_t *)ctx)->cc_framework_private maps to this structure. + */ +typedef struct kcf_context { + crypto_ctx_t kc_glbl_ctx; + uint_t kc_refcnt; + kmutex_t kc_in_use_lock; + /* + * kc_req_chain_first and kc_req_chain_last are used to chain + * multiple async requests using the same context. They should be + * NULL for sync requests. + */ + kcf_areq_node_t *kc_req_chain_first; + kcf_areq_node_t *kc_req_chain_last; + kcf_provider_desc_t *kc_prov_desc; /* Prov. descriptor */ + kcf_provider_desc_t *kc_sw_prov_desc; /* Prov. descriptor */ + kcf_mech_entry_t *kc_mech; + struct kcf_context *kc_secondctx; /* for dual contexts */ +} kcf_context_t; + +/* + * Bump up the reference count on the framework private context. A + * global context or a request that references this structure should + * do a hold. + */ +#define KCF_CONTEXT_REFHOLD(ictx) { \ + atomic_add_32(&(ictx)->kc_refcnt, 1); \ + ASSERT((ictx)->kc_refcnt != 0); \ +} + +/* + * Decrement the reference count on the framework private context. + * When the last reference is released, the framework private + * context structure is freed along with the global context. + */ +#define KCF_CONTEXT_REFRELE(ictx) { \ + ASSERT((ictx)->kc_refcnt != 0); \ + membar_exit(); \ + if (atomic_add_32_nv(&(ictx)->kc_refcnt, -1) == 0) \ + kcf_free_context(ictx); \ +} + +/* + * Check if we can release the context now. In case of CRYPTO_QUEUED + * we do not release it as we can do it only after the provider notified + * us. In case of CRYPTO_BUSY, the client can retry the request using + * the context, so we do not release the context. + * + * This macro should be called only from the final routine in + * an init/update/final sequence. We do not release the context in case + * of update operations. We require the consumer to free it + * explicitly, in case it wants to abandon the operation. This is done + * as there may be mechanisms in ECB mode that can continue even if + * an operation on a block fails. + */ +#define KCF_CONTEXT_COND_RELEASE(rv, kcf_ctx) { \ + if (KCF_CONTEXT_DONE(rv)) \ + KCF_CONTEXT_REFRELE(kcf_ctx); \ +} + +/* + * This macro determines whether we're done with a context. + */ +#define KCF_CONTEXT_DONE(rv) \ + ((rv) != CRYPTO_QUEUED && (rv) != CRYPTO_BUSY && \ + (rv) != CRYPTO_BUFFER_TOO_SMALL) + +/* + * A crypto_ctx_template_t is internally a pointer to this struct + */ +typedef struct kcf_ctx_template { + crypto_kcf_provider_handle_t ct_prov_handle; /* provider handle */ + uint_t ct_generation; /* generation # */ + size_t ct_size; /* for freeing */ + crypto_spi_ctx_template_t ct_prov_tmpl; /* context template */ + /* from the SW prov */ +} kcf_ctx_template_t; + +/* + * Structure for pool of threads working on global software queue. + */ +typedef struct kcf_pool { + uint32_t kp_threads; /* Number of threads in pool */ + uint32_t kp_idlethreads; /* Idle threads in pool */ + uint32_t kp_blockedthreads; /* Blocked threads in pool */ + + /* + * cv & lock to monitor the condition when no threads + * are around. In this case the failover thread kicks in. + */ + kcondvar_t kp_nothr_cv; + kmutex_t kp_thread_lock; + + /* Userspace thread creator variables. */ + boolean_t kp_signal_create_thread; /* Create requested flag */ + int kp_nthrs; /* # of threads to create */ + boolean_t kp_user_waiting; /* Thread waiting for work */ + + /* + * cv & lock for the condition where more threads need to be + * created. kp_user_lock also protects the three fileds above. + */ + kcondvar_t kp_user_cv; /* Creator cond. variable */ + kmutex_t kp_user_lock; /* Creator lock */ +} kcf_pool_t; + + +/* + * State of a crypto bufcall element. + */ +typedef enum cbuf_state { + CBUF_FREE = 1, + CBUF_WAITING, + CBUF_RUNNING +} cbuf_state_t; + +/* + * Structure of a crypto bufcall element. + */ +typedef struct kcf_cbuf_elem { + /* + * lock and cv to wait for CBUF_RUNNING to be done + * kc_lock also protects kc_state. + */ + kmutex_t kc_lock; + kcondvar_t kc_cv; + cbuf_state_t kc_state; + + struct kcf_cbuf_elem *kc_next; + struct kcf_cbuf_elem *kc_prev; + + void (*kc_func)(void *arg); + void *kc_arg; +} kcf_cbuf_elem_t; + +/* + * State of a notify element. + */ +typedef enum ntfy_elem_state { + NTFY_WAITING = 1, + NTFY_RUNNING +} ntfy_elem_state_t; + +/* + * Structure of a notify list element. + */ +typedef struct kcf_ntfy_elem { + /* + * lock and cv to wait for NTFY_RUNNING to be done. + * kn_lock also protects kn_state. + */ + kmutex_t kn_lock; + kcondvar_t kn_cv; + ntfy_elem_state_t kn_state; + + struct kcf_ntfy_elem *kn_next; + struct kcf_ntfy_elem *kn_prev; + + crypto_notify_callback_t kn_func; + uint32_t kn_event_mask; +} kcf_ntfy_elem_t; + + +/* + * The following values are based on the assumption that it would + * take around eight cpus to load a hardware provider (This is true for + * at least one product) and a kernel client may come from different + * low-priority interrupt levels. We will have CYRPTO_TASKQ_MIN number + * of cached taskq entries. The CRYPTO_TASKQ_MAX number is based on + * a throughput of 1GB/s using 512-byte buffers. These are just + * reasonable estimates and might need to change in future. + */ +#define CRYPTO_TASKQ_THREADS 8 +#define CYRPTO_TASKQ_MIN 64 +#define CRYPTO_TASKQ_MAX 2 * 1024 * 1024 + +extern int crypto_taskq_threads; +extern int crypto_taskq_minalloc; +extern int crypto_taskq_maxalloc; +extern kcf_global_swq_t *gswq; +extern int kcf_maxthreads; +extern int kcf_minthreads; + +/* + * All pending crypto bufcalls are put on a list. cbuf_list_lock + * protects changes to this list. + */ +extern kmutex_t cbuf_list_lock; +extern kcondvar_t cbuf_list_cv; + +/* + * All event subscribers are put on a list. kcf_notify_list_lock + * protects changes to this list. + */ +extern kmutex_t ntfy_list_lock; +extern kcondvar_t ntfy_list_cv; + +boolean_t kcf_get_next_logical_provider_member(kcf_provider_desc_t *, + kcf_provider_desc_t *, kcf_provider_desc_t **); +extern int kcf_get_hardware_provider(crypto_mech_type_t, crypto_mech_type_t, + boolean_t, kcf_provider_desc_t *, kcf_provider_desc_t **, + crypto_func_group_t); +extern int kcf_get_hardware_provider_nomech(offset_t, offset_t, + boolean_t, kcf_provider_desc_t *, kcf_provider_desc_t **); +extern void kcf_free_triedlist(kcf_prov_tried_t *); +extern kcf_prov_tried_t *kcf_insert_triedlist(kcf_prov_tried_t **, + kcf_provider_desc_t *, int); +extern kcf_provider_desc_t *kcf_get_mech_provider(crypto_mech_type_t, + kcf_mech_entry_t **, int *, kcf_prov_tried_t *, crypto_func_group_t, + boolean_t, size_t); +extern kcf_provider_desc_t *kcf_get_dual_provider(crypto_mechanism_t *, + crypto_mechanism_t *, kcf_mech_entry_t **, crypto_mech_type_t *, + crypto_mech_type_t *, int *, kcf_prov_tried_t *, + crypto_func_group_t, crypto_func_group_t, boolean_t, size_t); +extern crypto_ctx_t *kcf_new_ctx(crypto_call_req_t *, kcf_provider_desc_t *, + crypto_session_id_t); +extern int kcf_submit_request(kcf_provider_desc_t *, crypto_ctx_t *, + crypto_call_req_t *, kcf_req_params_t *, boolean_t); +extern void kcf_sched_destroy(void); +extern void kcf_sched_init(void); +extern void kcf_sched_start(void); +extern void kcf_sop_done(kcf_sreq_node_t *, int); +extern void kcf_aop_done(kcf_areq_node_t *, int); +extern int common_submit_request(kcf_provider_desc_t *, + crypto_ctx_t *, kcf_req_params_t *, crypto_req_handle_t); +extern void kcf_free_context(kcf_context_t *); + +extern int kcf_svc_wait(int *); +extern int kcf_svc_do_run(void); +extern int kcf_need_signature_verification(kcf_provider_desc_t *); +extern void kcf_verify_signature(void *); +extern struct modctl *kcf_get_modctl(crypto_provider_info_t *); +extern void verify_unverified_providers(void); +extern void kcf_free_req(kcf_areq_node_t *areq); +extern void crypto_bufcall_service(void); + +extern void kcf_walk_ntfylist(uint32_t, void *); +extern void kcf_do_notify(kcf_provider_desc_t *, boolean_t); + +extern kcf_dual_req_t *kcf_alloc_req(crypto_call_req_t *); +extern void kcf_next_req(void *, int); +extern void kcf_last_req(void *, int); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRYPTO_SCHED_IMPL_H */ diff --git a/module/icp/include/sys/crypto/spi.h b/module/icp/include/sys/crypto/spi.h new file mode 100644 index 000000000000..b4d6467f92e1 --- /dev/null +++ b/module/icp/include/sys/crypto/spi.h @@ -0,0 +1,721 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_CRYPTO_SPI_H +#define _SYS_CRYPTO_SPI_H + +/* + * CSPI: Cryptographic Service Provider Interface. + */ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +#define CRYPTO_SPI_VERSION_1 1 +#define CRYPTO_SPI_VERSION_2 2 +#define CRYPTO_SPI_VERSION_3 3 + +/* + * Provider-private handle. This handle is specified by a provider + * when it registers by means of the pi_provider_handle field of + * the crypto_provider_info structure, and passed to the provider + * when its entry points are invoked. + */ +typedef void *crypto_provider_handle_t; + +/* + * Context templates can be used to by software providers to pre-process + * keying material, such as key schedules. They are allocated by + * a software provider create_ctx_template(9E) entry point, and passed + * as argument to initialization and atomic provider entry points. + */ +typedef void *crypto_spi_ctx_template_t; + +/* + * Request handles are used by the kernel to identify an asynchronous + * request being processed by a provider. It is passed by the kernel + * to a hardware provider when submitting a request, and must be + * specified by a provider when calling crypto_op_notification(9F) + */ +typedef void *crypto_req_handle_t; + +/* Values for cc_flags field */ +#define CRYPTO_INIT_OPSTATE 0x00000001 /* allocate and init cc_opstate */ +#define CRYPTO_USE_OPSTATE 0x00000002 /* .. start using it as context */ + +/* + * The context structure is passed from the kernel to a provider. + * It contains the information needed to process a multi-part or + * single part operation. The context structure is not used + * by atomic operations. + * + * Parameters needed to perform a cryptographic operation, such + * as keys, mechanisms, input and output buffers, are passed + * as separate arguments to Provider routines. + */ +typedef struct crypto_ctx { + crypto_provider_handle_t cc_provider; + crypto_session_id_t cc_session; + void *cc_provider_private; /* owned by provider */ + void *cc_framework_private; /* owned by framework */ + uint32_t cc_flags; /* flags */ + void *cc_opstate; /* state */ +} crypto_ctx_t; + +/* + * Extended provider information. + */ + +/* + * valid values for ei_flags field of extended info structure + * They match the RSA Security, Inc PKCS#11 tokenInfo flags. + */ +#define CRYPTO_EXTF_RNG 0x00000001 +#define CRYPTO_EXTF_WRITE_PROTECTED 0x00000002 +#define CRYPTO_EXTF_LOGIN_REQUIRED 0x00000004 +#define CRYPTO_EXTF_USER_PIN_INITIALIZED 0x00000008 +#define CRYPTO_EXTF_CLOCK_ON_TOKEN 0x00000040 +#define CRYPTO_EXTF_PROTECTED_AUTHENTICATION_PATH 0x00000100 +#define CRYPTO_EXTF_DUAL_CRYPTO_OPERATIONS 0x00000200 +#define CRYPTO_EXTF_TOKEN_INITIALIZED 0x00000400 +#define CRYPTO_EXTF_USER_PIN_COUNT_LOW 0x00010000 +#define CRYPTO_EXTF_USER_PIN_FINAL_TRY 0x00020000 +#define CRYPTO_EXTF_USER_PIN_LOCKED 0x00040000 +#define CRYPTO_EXTF_USER_PIN_TO_BE_CHANGED 0x00080000 +#define CRYPTO_EXTF_SO_PIN_COUNT_LOW 0x00100000 +#define CRYPTO_EXTF_SO_PIN_FINAL_TRY 0x00200000 +#define CRYPTO_EXTF_SO_PIN_LOCKED 0x00400000 +#define CRYPTO_EXTF_SO_PIN_TO_BE_CHANGED 0x00800000 + +/* + * The crypto_control_ops structure contains pointers to control + * operations for cryptographic providers. It is passed through + * the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_control_ops { + void (*provider_status)(crypto_provider_handle_t, uint_t *); +} crypto_control_ops_t; + +/* + * The crypto_ctx_ops structure contains points to context and context + * templates management operations for cryptographic providers. It is + * passed through the crypto_ops(9S) structure when providers register + * with the kernel using crypto_register_provider(9F). + */ +typedef struct crypto_ctx_ops { + int (*create_ctx_template)(crypto_provider_handle_t, + crypto_mechanism_t *, crypto_key_t *, + crypto_spi_ctx_template_t *, size_t *, crypto_req_handle_t); + int (*free_context)(crypto_ctx_t *); +} crypto_ctx_ops_t; + +/* + * The crypto_digest_ops structure contains pointers to digest + * operations for cryptographic providers. It is passed through + * the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_digest_ops { + int (*digest_init)(crypto_ctx_t *, crypto_mechanism_t *, + crypto_req_handle_t); + int (*digest)(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, + crypto_req_handle_t); + int (*digest_update)(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); + int (*digest_key)(crypto_ctx_t *, crypto_key_t *, crypto_req_handle_t); + int (*digest_final)(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); + int (*digest_atomic)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_data_t *, + crypto_data_t *, crypto_req_handle_t); +} crypto_digest_ops_t; + +/* + * The crypto_cipher_ops structure contains pointers to encryption + * and decryption operations for cryptographic providers. It is + * passed through the crypto_ops(9S) structure when providers register + * with the kernel using crypto_register_provider(9F). + */ +typedef struct crypto_cipher_ops { + int (*encrypt_init)(crypto_ctx_t *, + crypto_mechanism_t *, crypto_key_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); + int (*encrypt)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*encrypt_update)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*encrypt_final)(crypto_ctx_t *, + crypto_data_t *, crypto_req_handle_t); + int (*encrypt_atomic)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); + + int (*decrypt_init)(crypto_ctx_t *, + crypto_mechanism_t *, crypto_key_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); + int (*decrypt)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*decrypt_update)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*decrypt_final)(crypto_ctx_t *, + crypto_data_t *, crypto_req_handle_t); + int (*decrypt_atomic)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); +} crypto_cipher_ops_t; + +/* + * The crypto_mac_ops structure contains pointers to MAC + * operations for cryptographic providers. It is passed through + * the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_mac_ops { + int (*mac_init)(crypto_ctx_t *, + crypto_mechanism_t *, crypto_key_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); + int (*mac)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*mac_update)(crypto_ctx_t *, + crypto_data_t *, crypto_req_handle_t); + int (*mac_final)(crypto_ctx_t *, + crypto_data_t *, crypto_req_handle_t); + int (*mac_atomic)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_data_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); + int (*mac_verify_atomic)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_data_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); +} crypto_mac_ops_t; + +/* + * The crypto_sign_ops structure contains pointers to signing + * operations for cryptographic providers. It is passed through + * the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_sign_ops { + int (*sign_init)(crypto_ctx_t *, + crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); + int (*sign)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*sign_update)(crypto_ctx_t *, + crypto_data_t *, crypto_req_handle_t); + int (*sign_final)(crypto_ctx_t *, + crypto_data_t *, crypto_req_handle_t); + int (*sign_atomic)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_data_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); + int (*sign_recover_init)(crypto_ctx_t *, crypto_mechanism_t *, + crypto_key_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); + int (*sign_recover)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*sign_recover_atomic)(crypto_provider_handle_t, + crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, + crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); +} crypto_sign_ops_t; + +/* + * The crypto_verify_ops structure contains pointers to verify + * operations for cryptographic providers. It is passed through + * the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_verify_ops { + int (*verify_init)(crypto_ctx_t *, + crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); + int (*do_verify)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*verify_update)(crypto_ctx_t *, + crypto_data_t *, crypto_req_handle_t); + int (*verify_final)(crypto_ctx_t *, + crypto_data_t *, crypto_req_handle_t); + int (*verify_atomic)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_data_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); + int (*verify_recover_init)(crypto_ctx_t *, crypto_mechanism_t *, + crypto_key_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); + int (*verify_recover)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*verify_recover_atomic)(crypto_provider_handle_t, + crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, + crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, + crypto_req_handle_t); +} crypto_verify_ops_t; + +/* + * The crypto_dual_ops structure contains pointers to dual + * cipher and sign/verify operations for cryptographic providers. + * It is passed through the crypto_ops(9S) structure when + * providers register with the kernel using + * crypto_register_provider(9F). + */ +typedef struct crypto_dual_ops { + int (*digest_encrypt_update)( + crypto_ctx_t *, crypto_ctx_t *, crypto_data_t *, + crypto_data_t *, crypto_req_handle_t); + int (*decrypt_digest_update)( + crypto_ctx_t *, crypto_ctx_t *, crypto_data_t *, + crypto_data_t *, crypto_req_handle_t); + int (*sign_encrypt_update)( + crypto_ctx_t *, crypto_ctx_t *, crypto_data_t *, + crypto_data_t *, crypto_req_handle_t); + int (*decrypt_verify_update)( + crypto_ctx_t *, crypto_ctx_t *, crypto_data_t *, + crypto_data_t *, crypto_req_handle_t); +} crypto_dual_ops_t; + +/* + * The crypto_dual_cipher_mac_ops structure contains pointers to dual + * cipher and MAC operations for cryptographic providers. + * It is passed through the crypto_ops(9S) structure when + * providers register with the kernel using + * crypto_register_provider(9F). + */ +typedef struct crypto_dual_cipher_mac_ops { + int (*encrypt_mac_init)(crypto_ctx_t *, + crypto_mechanism_t *, crypto_key_t *, crypto_mechanism_t *, + crypto_key_t *, crypto_spi_ctx_template_t, + crypto_spi_ctx_template_t, crypto_req_handle_t); + int (*encrypt_mac)(crypto_ctx_t *, + crypto_data_t *, crypto_dual_data_t *, crypto_data_t *, + crypto_req_handle_t); + int (*encrypt_mac_update)(crypto_ctx_t *, + crypto_data_t *, crypto_dual_data_t *, crypto_req_handle_t); + int (*encrypt_mac_final)(crypto_ctx_t *, + crypto_dual_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*encrypt_mac_atomic)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_mechanism_t *, + crypto_key_t *, crypto_data_t *, crypto_dual_data_t *, + crypto_data_t *, crypto_spi_ctx_template_t, + crypto_spi_ctx_template_t, crypto_req_handle_t); + + int (*mac_decrypt_init)(crypto_ctx_t *, + crypto_mechanism_t *, crypto_key_t *, crypto_mechanism_t *, + crypto_key_t *, crypto_spi_ctx_template_t, + crypto_spi_ctx_template_t, crypto_req_handle_t); + int (*mac_decrypt)(crypto_ctx_t *, + crypto_dual_data_t *, crypto_data_t *, crypto_data_t *, + crypto_req_handle_t); + int (*mac_decrypt_update)(crypto_ctx_t *, + crypto_dual_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*mac_decrypt_final)(crypto_ctx_t *, + crypto_data_t *, crypto_data_t *, crypto_req_handle_t); + int (*mac_decrypt_atomic)(crypto_provider_handle_t, + crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, + crypto_mechanism_t *, crypto_key_t *, crypto_dual_data_t *, + crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, + crypto_spi_ctx_template_t, crypto_req_handle_t); + int (*mac_verify_decrypt_atomic)(crypto_provider_handle_t, + crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, + crypto_mechanism_t *, crypto_key_t *, crypto_dual_data_t *, + crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, + crypto_spi_ctx_template_t, crypto_req_handle_t); +} crypto_dual_cipher_mac_ops_t; + +/* + * The crypto_random_number_ops structure contains pointers to random + * number operations for cryptographic providers. It is passed through + * the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_random_number_ops { + int (*seed_random)(crypto_provider_handle_t, crypto_session_id_t, + uchar_t *, size_t, uint_t, uint32_t, crypto_req_handle_t); + int (*generate_random)(crypto_provider_handle_t, crypto_session_id_t, + uchar_t *, size_t, crypto_req_handle_t); +} crypto_random_number_ops_t; + +/* + * Flag values for seed_random. + */ +#define CRYPTO_SEED_NOW 0x00000001 + +/* + * The crypto_session_ops structure contains pointers to session + * operations for cryptographic providers. It is passed through + * the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_session_ops { + int (*session_open)(crypto_provider_handle_t, crypto_session_id_t *, + crypto_req_handle_t); + int (*session_close)(crypto_provider_handle_t, crypto_session_id_t, + crypto_req_handle_t); + int (*session_login)(crypto_provider_handle_t, crypto_session_id_t, + crypto_user_type_t, char *, size_t, crypto_req_handle_t); + int (*session_logout)(crypto_provider_handle_t, crypto_session_id_t, + crypto_req_handle_t); +} crypto_session_ops_t; + +/* + * The crypto_object_ops structure contains pointers to object + * operations for cryptographic providers. It is passed through + * the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_object_ops { + int (*object_create)(crypto_provider_handle_t, crypto_session_id_t, + crypto_object_attribute_t *, uint_t, crypto_object_id_t *, + crypto_req_handle_t); + int (*object_copy)(crypto_provider_handle_t, crypto_session_id_t, + crypto_object_id_t, crypto_object_attribute_t *, uint_t, + crypto_object_id_t *, crypto_req_handle_t); + int (*object_destroy)(crypto_provider_handle_t, crypto_session_id_t, + crypto_object_id_t, crypto_req_handle_t); + int (*object_get_size)(crypto_provider_handle_t, crypto_session_id_t, + crypto_object_id_t, size_t *, crypto_req_handle_t); + int (*object_get_attribute_value)(crypto_provider_handle_t, + crypto_session_id_t, crypto_object_id_t, + crypto_object_attribute_t *, uint_t, crypto_req_handle_t); + int (*object_set_attribute_value)(crypto_provider_handle_t, + crypto_session_id_t, crypto_object_id_t, + crypto_object_attribute_t *, uint_t, crypto_req_handle_t); + int (*object_find_init)(crypto_provider_handle_t, crypto_session_id_t, + crypto_object_attribute_t *, uint_t, void **, + crypto_req_handle_t); + int (*object_find)(crypto_provider_handle_t, void *, + crypto_object_id_t *, uint_t, uint_t *, crypto_req_handle_t); + int (*object_find_final)(crypto_provider_handle_t, void *, + crypto_req_handle_t); +} crypto_object_ops_t; + +/* + * The crypto_key_ops structure contains pointers to key + * operations for cryptographic providers. It is passed through + * the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_key_ops { + int (*key_generate)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_object_attribute_t *, uint_t, + crypto_object_id_t *, crypto_req_handle_t); + int (*key_generate_pair)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_object_attribute_t *, uint_t, + crypto_object_attribute_t *, uint_t, crypto_object_id_t *, + crypto_object_id_t *, crypto_req_handle_t); + int (*key_wrap)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_object_id_t *, + uchar_t *, size_t *, crypto_req_handle_t); + int (*key_unwrap)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, uchar_t *, size_t *, + crypto_object_attribute_t *, uint_t, + crypto_object_id_t *, crypto_req_handle_t); + int (*key_derive)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_object_attribute_t *, + uint_t, crypto_object_id_t *, crypto_req_handle_t); + int (*key_check)(crypto_provider_handle_t, crypto_mechanism_t *, + crypto_key_t *); +} crypto_key_ops_t; + +/* + * The crypto_provider_management_ops structure contains pointers + * to management operations for cryptographic providers. It is passed + * through the crypto_ops(9S) structure when providers register with the + * kernel using crypto_register_provider(9F). + */ +typedef struct crypto_provider_management_ops { + int (*ext_info)(crypto_provider_handle_t, + crypto_provider_ext_info_t *, crypto_req_handle_t); + int (*init_token)(crypto_provider_handle_t, char *, size_t, + char *, crypto_req_handle_t); + int (*init_pin)(crypto_provider_handle_t, crypto_session_id_t, + char *, size_t, crypto_req_handle_t); + int (*set_pin)(crypto_provider_handle_t, crypto_session_id_t, + char *, size_t, char *, size_t, crypto_req_handle_t); +} crypto_provider_management_ops_t; + +typedef struct crypto_mech_ops { + int (*copyin_mechanism)(crypto_provider_handle_t, + crypto_mechanism_t *, crypto_mechanism_t *, int *, int); + int (*copyout_mechanism)(crypto_provider_handle_t, + crypto_mechanism_t *, crypto_mechanism_t *, int *, int); + int (*free_mechanism)(crypto_provider_handle_t, crypto_mechanism_t *); +} crypto_mech_ops_t; + +typedef struct crypto_nostore_key_ops { + int (*nostore_key_generate)(crypto_provider_handle_t, + crypto_session_id_t, crypto_mechanism_t *, + crypto_object_attribute_t *, uint_t, crypto_object_attribute_t *, + uint_t, crypto_req_handle_t); + int (*nostore_key_generate_pair)(crypto_provider_handle_t, + crypto_session_id_t, crypto_mechanism_t *, + crypto_object_attribute_t *, uint_t, crypto_object_attribute_t *, + uint_t, crypto_object_attribute_t *, uint_t, + crypto_object_attribute_t *, uint_t, crypto_req_handle_t); + int (*nostore_key_derive)(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_object_attribute_t *, + uint_t, crypto_object_attribute_t *, uint_t, crypto_req_handle_t); +} crypto_nostore_key_ops_t; + +/* + * The crypto_ops(9S) structure contains the structures containing + * the pointers to functions implemented by cryptographic providers. + * It is specified as part of the crypto_provider_info(9S) + * supplied by a provider when it registers with the kernel + * by calling crypto_register_provider(9F). + */ +typedef struct crypto_ops_v1 { + crypto_control_ops_t *co_control_ops; + crypto_digest_ops_t *co_digest_ops; + crypto_cipher_ops_t *co_cipher_ops; + crypto_mac_ops_t *co_mac_ops; + crypto_sign_ops_t *co_sign_ops; + crypto_verify_ops_t *co_verify_ops; + crypto_dual_ops_t *co_dual_ops; + crypto_dual_cipher_mac_ops_t *co_dual_cipher_mac_ops; + crypto_random_number_ops_t *co_random_ops; + crypto_session_ops_t *co_session_ops; + crypto_object_ops_t *co_object_ops; + crypto_key_ops_t *co_key_ops; + crypto_provider_management_ops_t *co_provider_ops; + crypto_ctx_ops_t *co_ctx_ops; +} crypto_ops_v1_t; + +typedef struct crypto_ops_v2 { + crypto_ops_v1_t v1_ops; + crypto_mech_ops_t *co_mech_ops; +} crypto_ops_v2_t; + +typedef struct crypto_ops_v3 { + crypto_ops_v2_t v2_ops; + crypto_nostore_key_ops_t *co_nostore_key_ops; +} crypto_ops_v3_t; + +typedef struct crypto_ops { + union { + crypto_ops_v3_t cou_v3; + crypto_ops_v2_t cou_v2; + crypto_ops_v1_t cou_v1; + } cou; +} crypto_ops_t; + +#define co_control_ops cou.cou_v1.co_control_ops +#define co_digest_ops cou.cou_v1.co_digest_ops +#define co_cipher_ops cou.cou_v1.co_cipher_ops +#define co_mac_ops cou.cou_v1.co_mac_ops +#define co_sign_ops cou.cou_v1.co_sign_ops +#define co_verify_ops cou.cou_v1.co_verify_ops +#define co_dual_ops cou.cou_v1.co_dual_ops +#define co_dual_cipher_mac_ops cou.cou_v1.co_dual_cipher_mac_ops +#define co_random_ops cou.cou_v1.co_random_ops +#define co_session_ops cou.cou_v1.co_session_ops +#define co_object_ops cou.cou_v1.co_object_ops +#define co_key_ops cou.cou_v1.co_key_ops +#define co_provider_ops cou.cou_v1.co_provider_ops +#define co_ctx_ops cou.cou_v1.co_ctx_ops +#define co_mech_ops cou.cou_v2.co_mech_ops +#define co_nostore_key_ops cou.cou_v3.co_nostore_key_ops + +/* + * The mechanism info structure crypto_mech_info_t contains a function group + * bit mask cm_func_group_mask. This field, of type crypto_func_group_t, + * specifies the provider entry point that can be used a particular + * mechanism. The function group mask is a combination of the following values. + */ + +typedef uint32_t crypto_func_group_t; + + +#define CRYPTO_FG_ENCRYPT 0x00000001 /* encrypt_init() */ +#define CRYPTO_FG_DECRYPT 0x00000002 /* decrypt_init() */ +#define CRYPTO_FG_DIGEST 0x00000004 /* digest_init() */ +#define CRYPTO_FG_SIGN 0x00000008 /* sign_init() */ +#define CRYPTO_FG_SIGN_RECOVER 0x00000010 /* sign_recover_init() */ +#define CRYPTO_FG_VERIFY 0x00000020 /* verify_init() */ +#define CRYPTO_FG_VERIFY_RECOVER 0x00000040 /* verify_recover_init() */ +#define CRYPTO_FG_GENERATE 0x00000080 /* key_generate() */ +#define CRYPTO_FG_GENERATE_KEY_PAIR 0x00000100 /* key_generate_pair() */ +#define CRYPTO_FG_WRAP 0x00000200 /* key_wrap() */ +#define CRYPTO_FG_UNWRAP 0x00000400 /* key_unwrap() */ +#define CRYPTO_FG_DERIVE 0x00000800 /* key_derive() */ +#define CRYPTO_FG_MAC 0x00001000 /* mac_init() */ +#define CRYPTO_FG_ENCRYPT_MAC 0x00002000 /* encrypt_mac_init() */ +#define CRYPTO_FG_MAC_DECRYPT 0x00004000 /* decrypt_mac_init() */ +#define CRYPTO_FG_ENCRYPT_ATOMIC 0x00008000 /* encrypt_atomic() */ +#define CRYPTO_FG_DECRYPT_ATOMIC 0x00010000 /* decrypt_atomic() */ +#define CRYPTO_FG_MAC_ATOMIC 0x00020000 /* mac_atomic() */ +#define CRYPTO_FG_DIGEST_ATOMIC 0x00040000 /* digest_atomic() */ +#define CRYPTO_FG_SIGN_ATOMIC 0x00080000 /* sign_atomic() */ +#define CRYPTO_FG_SIGN_RECOVER_ATOMIC 0x00100000 /* sign_recover_atomic() */ +#define CRYPTO_FG_VERIFY_ATOMIC 0x00200000 /* verify_atomic() */ +#define CRYPTO_FG_VERIFY_RECOVER_ATOMIC 0x00400000 /* verify_recover_atomic() */ +#define CRYPTO_FG_ENCRYPT_MAC_ATOMIC 0x00800000 /* encrypt_mac_atomic() */ +#define CRYPTO_FG_MAC_DECRYPT_ATOMIC 0x01000000 /* mac_decrypt_atomic() */ +#define CRYPTO_FG_RESERVED 0x80000000 + +/* + * Maximum length of the pi_provider_description field of the + * crypto_provider_info structure. + */ +#define CRYPTO_PROVIDER_DESCR_MAX_LEN 64 + + +/* Bit mask for all the simple operations */ +#define CRYPTO_FG_SIMPLEOP_MASK (CRYPTO_FG_ENCRYPT | CRYPTO_FG_DECRYPT | \ + CRYPTO_FG_DIGEST | CRYPTO_FG_SIGN | CRYPTO_FG_VERIFY | CRYPTO_FG_MAC | \ + CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT_ATOMIC | \ + CRYPTO_FG_MAC_ATOMIC | CRYPTO_FG_DIGEST_ATOMIC | CRYPTO_FG_SIGN_ATOMIC | \ + CRYPTO_FG_VERIFY_ATOMIC) + +/* Bit mask for all the dual operations */ +#define CRYPTO_FG_MAC_CIPHER_MASK (CRYPTO_FG_ENCRYPT_MAC | \ + CRYPTO_FG_MAC_DECRYPT | CRYPTO_FG_ENCRYPT_MAC_ATOMIC | \ + CRYPTO_FG_MAC_DECRYPT_ATOMIC) + +/* Add other combos to CRYPTO_FG_DUAL_MASK */ +#define CRYPTO_FG_DUAL_MASK CRYPTO_FG_MAC_CIPHER_MASK + +/* + * The crypto_mech_info structure specifies one of the mechanisms + * supported by a cryptographic provider. The pi_mechanisms field of + * the crypto_provider_info structure contains a pointer to an array + * of crypto_mech_info's. + */ +typedef struct crypto_mech_info { + crypto_mech_name_t cm_mech_name; + crypto_mech_type_t cm_mech_number; + crypto_func_group_t cm_func_group_mask; + ssize_t cm_min_key_length; + ssize_t cm_max_key_length; + uint32_t cm_mech_flags; +} crypto_mech_info_t; + +/* Alias the old name to the new name for compatibility. */ +#define cm_keysize_unit cm_mech_flags + +/* + * The following is used by a provider that sets + * CRYPTO_HASH_NO_UPDATE. It needs to specify the maximum + * input data size it can digest in this field. + */ +#define cm_max_input_length cm_max_key_length + +/* + * crypto_kcf_provider_handle_t is a handle allocated by the kernel. + * It is returned after the provider registers with + * crypto_register_provider(), and must be specified by the provider + * when calling crypto_unregister_provider(), and + * crypto_provider_notification(). + */ +typedef uint_t crypto_kcf_provider_handle_t; + +/* + * Provider information. Passed as argument to crypto_register_provider(9F). + * Describes the provider and its capabilities. Multiple providers can + * register for the same device instance. In this case, the same + * pi_provider_dev must be specified with a different pi_provider_handle. + */ +typedef struct crypto_provider_info_v1 { + uint_t pi_interface_version; + char *pi_provider_description; + crypto_provider_type_t pi_provider_type; + crypto_provider_handle_t pi_provider_handle; + crypto_ops_t *pi_ops_vector; + uint_t pi_mech_list_count; + crypto_mech_info_t *pi_mechanisms; + uint_t pi_logical_provider_count; + crypto_kcf_provider_handle_t *pi_logical_providers; +} crypto_provider_info_v1_t; + +typedef struct crypto_provider_info_v2 { + crypto_provider_info_v1_t v1_info; + uint_t pi_flags; +} crypto_provider_info_v2_t; + +typedef struct crypto_provider_info { + union { + crypto_provider_info_v2_t piu_v2; + crypto_provider_info_v1_t piu_v1; + } piu; +} crypto_provider_info_t; + +#define pi_interface_version piu.piu_v1.pi_interface_version +#define pi_provider_description piu.piu_v1.pi_provider_description +#define pi_provider_type piu.piu_v1.pi_provider_type +#define pi_provider_handle piu.piu_v1.pi_provider_handle +#define pi_ops_vector piu.piu_v1.pi_ops_vector +#define pi_mech_list_count piu.piu_v1.pi_mech_list_count +#define pi_mechanisms piu.piu_v1.pi_mechanisms +#define pi_logical_provider_count piu.piu_v1.pi_logical_provider_count +#define pi_logical_providers piu.piu_v1.pi_logical_providers +#define pi_flags piu.piu_v2.pi_flags + +/* hidden providers can only be accessed via a logical provider */ +#define CRYPTO_HIDE_PROVIDER 0x00000001 +/* + * provider can not do multi-part digest (updates) and has a limit + * on maximum input data that it can digest. + */ +#define CRYPTO_HASH_NO_UPDATE 0x00000002 + +/* provider can handle the request without returning a CRYPTO_QUEUED */ +#define CRYPTO_SYNCHRONOUS 0x00000004 + +#define CRYPTO_PIFLAGS_RESERVED2 0x40000000 +#define CRYPTO_PIFLAGS_RESERVED1 0x80000000 + +/* + * Provider status passed by a provider to crypto_provider_notification(9F) + * and returned by the provider_stauts(9E) entry point. + */ +#define CRYPTO_PROVIDER_READY 0 +#define CRYPTO_PROVIDER_BUSY 1 +#define CRYPTO_PROVIDER_FAILED 2 + +/* + * Functions exported by Solaris to cryptographic providers. Providers + * call these functions to register and unregister, notify the kernel + * of state changes, and notify the kernel when a asynchronous request + * completed. + */ +extern int crypto_register_provider(crypto_provider_info_t *, + crypto_kcf_provider_handle_t *); +extern int crypto_unregister_provider(crypto_kcf_provider_handle_t); +extern void crypto_provider_notification(crypto_kcf_provider_handle_t, uint_t); +extern void crypto_op_notification(crypto_req_handle_t, int); +extern int crypto_kmflag(crypto_req_handle_t); + + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_CRYPTO_SPI_H */ diff --git a/module/icp/include/sys/ia32/asm_linkage.h b/module/icp/include/sys/ia32/asm_linkage.h new file mode 100644 index 000000000000..f2dae7093b94 --- /dev/null +++ b/module/icp/include/sys/ia32/asm_linkage.h @@ -0,0 +1,307 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _IA32_SYS_ASM_LINKAGE_H +#define _IA32_SYS_ASM_LINKAGE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef _ASM /* The remainder of this file is only for assembly files */ + +/* + * make annoying differences in assembler syntax go away + */ + +/* + * D16 and A16 are used to insert instructions prefixes; the + * macros help the assembler code be slightly more portable. + */ +#if !defined(__GNUC_AS__) +/* + * /usr/ccs/bin/as prefixes are parsed as separate instructions + */ +#define D16 data16; +#define A16 addr16; + +/* + * (There are some weird constructs in constant expressions) + */ +#define _CONST(const) [const] +#define _BITNOT(const) -1!_CONST(const) +#define _MUL(a, b) _CONST(a \* b) + +#else +/* + * Why not use the 'data16' and 'addr16' prefixes .. well, the + * assembler doesn't quite believe in real mode, and thus argues with + * us about what we're trying to do. + */ +#define D16 .byte 0x66; +#define A16 .byte 0x67; + +#define _CONST(const) (const) +#define _BITNOT(const) ~_CONST(const) +#define _MUL(a, b) _CONST(a * b) + +#endif + +/* + * C pointers are different sizes between i386 and amd64. + * These constants can be used to compute offsets into pointer arrays. + */ +#if defined(__amd64) +#define CLONGSHIFT 3 +#define CLONGSIZE 8 +#define CLONGMASK 7 +#elif defined(__i386) +#define CLONGSHIFT 2 +#define CLONGSIZE 4 +#define CLONGMASK 3 +#endif + +/* + * Since we know we're either ILP32 or LP64 .. + */ +#define CPTRSHIFT CLONGSHIFT +#define CPTRSIZE CLONGSIZE +#define CPTRMASK CLONGMASK + +#if CPTRSIZE != (1 << CPTRSHIFT) || CLONGSIZE != (1 << CLONGSHIFT) +#error "inconsistent shift constants" +#endif + +#if CPTRMASK != (CPTRSIZE - 1) || CLONGMASK != (CLONGSIZE - 1) +#error "inconsistent mask constants" +#endif + +#define ASM_ENTRY_ALIGN 16 + +/* + * SSE register alignment and save areas + */ + +#define XMM_SIZE 16 +#define XMM_ALIGN 16 + +#if defined(__amd64) + +#define SAVE_XMM_PROLOG(sreg, nreg) \ + subq $_CONST(_MUL(XMM_SIZE, nreg)), %rsp; \ + movq %rsp, sreg + +#define RSTOR_XMM_EPILOG(sreg, nreg) \ + addq $_CONST(_MUL(XMM_SIZE, nreg)), %rsp + +#elif defined(__i386) + +#define SAVE_XMM_PROLOG(sreg, nreg) \ + subl $_CONST(_MUL(XMM_SIZE, nreg) + XMM_ALIGN), %esp; \ + movl %esp, sreg; \ + addl $XMM_ALIGN, sreg; \ + andl $_BITNOT(XMM_ALIGN-1), sreg + +#define RSTOR_XMM_EPILOG(sreg, nreg) \ + addl $_CONST(_MUL(XMM_SIZE, nreg) + XMM_ALIGN), %esp; + +#endif /* __i386 */ + +/* + * profiling causes definitions of the MCOUNT and RTMCOUNT + * particular to the type + */ +#ifdef GPROF + +#define MCOUNT(x) \ + pushl %ebp; \ + movl %esp, %ebp; \ + call _mcount; \ + popl %ebp + +#endif /* GPROF */ + +#ifdef PROF + +#define MCOUNT(x) \ +/* CSTYLED */ \ + .lcomm .L_/**/x/**/1, 4, 4; \ + pushl %ebp; \ + movl %esp, %ebp; \ +/* CSTYLED */ \ + movl $.L_/**/x/**/1, %edx; \ + call _mcount; \ + popl %ebp + +#endif /* PROF */ + +/* + * if we are not profiling, MCOUNT should be defined to nothing + */ +#if !defined(PROF) && !defined(GPROF) +#define MCOUNT(x) +#endif /* !defined(PROF) && !defined(GPROF) */ + +#define RTMCOUNT(x) MCOUNT(x) + +/* + * Macro to define weak symbol aliases. These are similar to the ANSI-C + * #pragma weak _name = name + * except a compiler can determine type. The assembler must be told. Hence, + * the second parameter must be the type of the symbol (i.e.: function,...) + */ +#define ANSI_PRAGMA_WEAK(sym, stype) \ +/* CSTYLED */ \ + .weak _/**/sym; \ +/* CSTYLED */ \ + .type _/**/sym, @stype; \ +/* CSTYLED */ \ +_/**/sym = sym + +/* + * Like ANSI_PRAGMA_WEAK(), but for unrelated names, as in: + * #pragma weak sym1 = sym2 + */ +#define ANSI_PRAGMA_WEAK2(sym1, sym2, stype) \ + .weak sym1; \ + .type sym1, @stype; \ +sym1 = sym2 + +/* + * ENTRY provides the standard procedure entry code and an easy way to + * insert the calls to mcount for profiling. ENTRY_NP is identical, but + * never calls mcount. + */ +#define ENTRY(x) \ + .text; \ + .align ASM_ENTRY_ALIGN; \ + .globl x; \ + .type x, @function; \ +x: MCOUNT(x) + +#define ENTRY_NP(x) \ + .text; \ + .align ASM_ENTRY_ALIGN; \ + .globl x; \ + .type x, @function; \ +x: + +#define RTENTRY(x) \ + .text; \ + .align ASM_ENTRY_ALIGN; \ + .globl x; \ + .type x, @function; \ +x: RTMCOUNT(x) + +/* + * ENTRY2 is identical to ENTRY but provides two labels for the entry point. + */ +#define ENTRY2(x, y) \ + .text; \ + .align ASM_ENTRY_ALIGN; \ + .globl x, y; \ + .type x, @function; \ + .type y, @function; \ +/* CSTYLED */ \ +x: ; \ +y: MCOUNT(x) + +#define ENTRY_NP2(x, y) \ + .text; \ + .align ASM_ENTRY_ALIGN; \ + .globl x, y; \ + .type x, @function; \ + .type y, @function; \ +/* CSTYLED */ \ +x: ; \ +y: + + +/* + * ALTENTRY provides for additional entry points. + */ +#define ALTENTRY(x) \ + .globl x; \ + .type x, @function; \ +x: + +/* + * DGDEF and DGDEF2 provide global data declarations. + * + * DGDEF provides a word aligned word of storage. + * + * DGDEF2 allocates "sz" bytes of storage with **NO** alignment. This + * implies this macro is best used for byte arrays. + * + * DGDEF3 allocates "sz" bytes of storage with "algn" alignment. + */ +#define DGDEF2(name, sz) \ + .data; \ + .globl name; \ + .type name, @object; \ + .size name, sz; \ +name: + +#define DGDEF3(name, sz, algn) \ + .data; \ + .align algn; \ + .globl name; \ + .type name, @object; \ + .size name, sz; \ +name: + +#define DGDEF(name) DGDEF3(name, 4, 4) + +/* + * SET_SIZE trails a function and set the size for the ELF symbol table. + */ +#define SET_SIZE(x) \ + .size x, [.-x] + +/* + * NWORD provides native word value. + */ +#if defined(__amd64) + +/*CSTYLED*/ +#define NWORD quad + +#elif defined(__i386) + +#define NWORD long + +#endif /* __i386 */ + +#endif /* _ASM */ + +#ifdef __cplusplus +} +#endif + +#endif /* _IA32_SYS_ASM_LINKAGE_H */ diff --git a/module/icp/include/sys/ia32/stack.h b/module/icp/include/sys/ia32/stack.h new file mode 100644 index 000000000000..c4deb7bcaf5a --- /dev/null +++ b/module/icp/include/sys/ia32/stack.h @@ -0,0 +1,160 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2004 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _IA32_SYS_STACK_H +#define _IA32_SYS_STACK_H + +#if !defined(_ASM) + +#include + +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * In the x86 world, a stack frame looks like this: + * + * |--------------------------| + * 4n+8(%ebp) ->| argument word n | + * | ... | (Previous frame) + * 8(%ebp) ->| argument word 0 | + * |--------------------------|-------------------- + * 4(%ebp) ->| return address | + * |--------------------------| + * 0(%ebp) ->| previous %ebp (optional) | + * |--------------------------| + * -4(%ebp) ->| unspecified | (Current frame) + * | ... | + * 0(%esp) ->| variable size | + * |--------------------------| + */ + +/* + * Stack alignment macros. + */ + +#define STACK_ALIGN32 4 +#define STACK_ENTRY_ALIGN32 4 +#define STACK_BIAS32 0 +#define SA32(x) (((x)+(STACK_ALIGN32-1)) & ~(STACK_ALIGN32-1)) +#define STACK_RESERVE32 0 +#define MINFRAME32 0 + +#if defined(__amd64) + +/* + * In the amd64 world, a stack frame looks like this: + * + * |--------------------------| + * 8n+16(%rbp)->| argument word n | + * | ... | (Previous frame) + * 16(%rbp) ->| argument word 0 | + * |--------------------------|-------------------- + * 8(%rbp) ->| return address | + * |--------------------------| + * 0(%rbp) ->| previous %rbp | + * |--------------------------| + * -8(%rbp) ->| unspecified | (Current frame) + * | ... | + * 0(%rsp) ->| variable size | + * |--------------------------| + * -128(%rsp) ->| reserved for function | + * |--------------------------| + * + * The end of the input argument area must be aligned on a 16-byte + * boundary; i.e. (%rsp - 8) % 16 == 0 at function entry. + * + * The 128-byte location beyond %rsp is considered to be reserved for + * functions and is NOT modified by signal handlers. It can be used + * to store temporary data that is not needed across function calls. + */ + +/* + * Stack alignment macros. + */ + +#define STACK_ALIGN64 16 +#define STACK_ENTRY_ALIGN64 8 +#define STACK_BIAS64 0 +#define SA64(x) (((x)+(STACK_ALIGN64-1)) & ~(STACK_ALIGN64-1)) +#define STACK_RESERVE64 128 +#define MINFRAME64 0 + +#define STACK_ALIGN STACK_ALIGN64 +#define STACK_ENTRY_ALIGN STACK_ENTRY_ALIGN64 +#define STACK_BIAS STACK_BIAS64 +#define SA(x) SA64(x) +#define STACK_RESERVE STACK_RESERVE64 +#define MINFRAME MINFRAME64 + +#elif defined(__i386) + +#define STACK_ALIGN STACK_ALIGN32 +#define STACK_ENTRY_ALIGN STACK_ENTRY_ALIGN32 +#define STACK_BIAS STACK_BIAS32 +#define SA(x) SA32(x) +#define STACK_RESERVE STACK_RESERVE32 +#define MINFRAME MINFRAME32 + +#endif /* __i386 */ + +#if defined(_KERNEL) && !defined(_ASM) + +#if defined(DEBUG) +#if STACK_ALIGN == 4 +#define ASSERT_STACK_ALIGNED() \ + { \ + uint32_t __tmp; \ + ASSERT((((uintptr_t)&__tmp) & (STACK_ALIGN - 1)) == 0); \ + } +#elif (STACK_ALIGN == 16) && (_LONG_DOUBLE_ALIGNMENT == 16) +#define ASSERT_STACK_ALIGNED() \ + { \ + long double __tmp; \ + ASSERT((((uintptr_t)&__tmp) & (STACK_ALIGN - 1)) == 0); \ + } +#endif +#else /* DEBUG */ +#define ASSERT_STACK_ALIGNED() +#endif /* DEBUG */ + +struct regs; + +void traceregs(struct regs *); +void traceback(caddr_t); + +#endif /* defined(_KERNEL) && !defined(_ASM) */ + +#define STACK_GROWTH_DOWN /* stacks grow from high to low addresses */ + +#ifdef __cplusplus +} +#endif + +#endif /* _IA32_SYS_STACK_H */ diff --git a/module/icp/include/sys/ia32/trap.h b/module/icp/include/sys/ia32/trap.h new file mode 100644 index 000000000000..55b94969b80b --- /dev/null +++ b/module/icp/include/sys/ia32/trap.h @@ -0,0 +1,107 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* Copyright (c) 1990, 1991 UNIX System Laboratories, Inc. */ +/* Copyright (c) 1984, 1986, 1987, 1988, 1989, 1990 AT&T */ +/* All Rights Reserved */ + +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _IA32_SYS_TRAP_H +#define _IA32_SYS_TRAP_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Trap type values + */ + +#define T_ZERODIV 0x0 /* #de divide by 0 error */ +#define T_SGLSTP 0x1 /* #db single step */ +#define T_NMIFLT 0x2 /* NMI */ +#define T_BPTFLT 0x3 /* #bp breakpoint fault, INT3 insn */ +#define T_OVFLW 0x4 /* #of INTO overflow fault */ +#define T_BOUNDFLT 0x5 /* #br BOUND insn fault */ +#define T_ILLINST 0x6 /* #ud invalid opcode fault */ +#define T_NOEXTFLT 0x7 /* #nm device not available: x87 */ +#define T_DBLFLT 0x8 /* #df double fault */ +#define T_EXTOVRFLT 0x9 /* [not generated: 386 only] */ +#define T_TSSFLT 0xa /* #ts invalid TSS fault */ +#define T_SEGFLT 0xb /* #np segment not present fault */ +#define T_STKFLT 0xc /* #ss stack fault */ +#define T_GPFLT 0xd /* #gp general protection fault */ +#define T_PGFLT 0xe /* #pf page fault */ +#define T_EXTERRFLT 0x10 /* #mf x87 FPU error fault */ +#define T_ALIGNMENT 0x11 /* #ac alignment check error */ +#define T_MCE 0x12 /* #mc machine check exception */ +#define T_SIMDFPE 0x13 /* #xm SSE/SSE exception */ +#define T_DBGENTR 0x14 /* debugger entry */ +#define T_ENDPERR 0x21 /* emulated extension error flt */ +#define T_ENOEXTFLT 0x20 /* emulated ext not present */ +#define T_FASTTRAP 0xd2 /* fast system call */ +#define T_SYSCALLINT 0x91 /* general system call */ +#define T_DTRACE_RET 0x7f /* DTrace pid return */ +#define T_INT80 0x80 /* int80 handler for linux emulation */ +#define T_SOFTINT 0x50fd /* pseudo softint trap type */ + +/* + * Pseudo traps. + */ +#define T_INTERRUPT 0x100 +#define T_FAULT 0x200 +#define T_AST 0x400 +#define T_SYSCALL 0x180 + + +/* + * Values of error code on stack in case of page fault + */ + +#define PF_ERR_MASK 0x01 /* Mask for error bit */ +#define PF_ERR_PAGE 0x00 /* page not present */ +#define PF_ERR_PROT 0x01 /* protection error */ +#define PF_ERR_WRITE 0x02 /* fault caused by write (else read) */ +#define PF_ERR_USER 0x04 /* processor was in user mode */ + /* (else supervisor) */ +#define PF_ERR_EXEC 0x10 /* attempt to execute a No eXec page (AMD) */ + +/* + * Definitions for fast system call subfunctions + */ +#define T_FNULL 0 /* Null trap for testing */ +#define T_FGETFP 1 /* Get emulated FP context */ +#define T_FSETFP 2 /* Set emulated FP context */ +#define T_GETHRTIME 3 /* Get high resolution time */ +#define T_GETHRVTIME 4 /* Get high resolution virtual time */ +#define T_GETHRESTIME 5 /* Get high resolution time */ +#define T_GETLGRP 6 /* Get home lgrpid */ + +#define T_LASTFAST 6 /* Last valid subfunction */ + +#ifdef __cplusplus +} +#endif + +#endif /* _IA32_SYS_TRAP_H */ diff --git a/module/icp/include/sys/modctl.h b/module/icp/include/sys/modctl.h new file mode 100644 index 000000000000..a0b94ef39db8 --- /dev/null +++ b/module/icp/include/sys/modctl.h @@ -0,0 +1,477 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_MODCTL_H +#define _SYS_MODCTL_H + +/* + * loadable module support. + */ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +struct modlmisc; +struct modlinkage; + +/* + * The following structure defines the operations used by modctl + * to load and unload modules. Each supported loadable module type + * requires a set of mod_ops. + */ +struct mod_ops { + int (*modm_install)(struct modlmisc *, struct modlinkage *); + int (*modm_remove)(struct modlmisc *, struct modlinkage *); + int (*modm_info)(void *, struct modlinkage *, int *); +}; + +/* + * The defined set of mod_ops structures for each loadable module type + * Defined in modctl.c + */ +extern struct mod_ops mod_brandops; +#if defined(__i386) || defined(__amd64) +extern struct mod_ops mod_cpuops; +#endif +extern struct mod_ops mod_cryptoops; +extern struct mod_ops mod_driverops; +extern struct mod_ops mod_execops; +extern struct mod_ops mod_fsops; +extern struct mod_ops mod_miscops; +extern struct mod_ops mod_schedops; +extern struct mod_ops mod_strmodops; +extern struct mod_ops mod_syscallops; +extern struct mod_ops mod_sockmodops; +#ifdef _SYSCALL32_IMPL +extern struct mod_ops mod_syscallops32; +#endif +extern struct mod_ops mod_dacfops; +extern struct mod_ops mod_ippops; +extern struct mod_ops mod_pcbeops; +extern struct mod_ops mod_devfsops; +extern struct mod_ops mod_kiconvops; + +/* + * Definitions for the module specific linkage structures. + * The first two fields are the same in all of the structures. + * The linkinfo is for informational purposes only and is returned by + * modctl with the MODINFO cmd. + */ + +/* For cryptographic providers */ +struct modlcrypto { + struct mod_ops *crypto_modops; + char *crypto_linkinfo; +}; + +/* For misc */ +struct modlmisc { + struct mod_ops *misc_modops; + char *misc_linkinfo; +}; + +/* + * Revision number of loadable modules support. This is the value + * that must be used in the modlinkage structure. + */ +#define MODREV_1 1 + +/* + * The modlinkage structure is the structure that the module writer + * provides to the routines to install, remove, and stat a module. + * The ml_linkage element is an array of pointers to linkage structures. + * For most modules there is only one linkage structure. We allocate + * enough space for 3 linkage structures which happens to be the most + * we have in any sun supplied module. For those modules with more + * than 3 linkage structures (which is very unlikely), a modlinkage + * structure must be kmem_alloc'd in the module wrapper to be big enough + * for all of the linkage structures. + */ +struct modlinkage { + int ml_rev; /* rev of loadable modules system */ +#ifdef _LP64 + void *ml_linkage[7]; /* more space in 64-bit OS */ +#else + void *ml_linkage[4]; /* NULL terminated list of */ + /* linkage structures */ +#endif +}; + +/* + * commands. These are the commands supported by the modctl system call. + */ +#define MODLOAD 0 +#define MODUNLOAD 1 +#define MODINFO 2 +#define MODRESERVED 3 +#define MODSETMINIROOT 4 +#define MODADDMAJBIND 5 +#define MODGETPATH 6 +#define MODREADSYSBIND 7 +#define MODGETMAJBIND 8 +#define MODGETNAME 9 +#define MODSIZEOF_DEVID 10 +#define MODGETDEVID 11 +#define MODSIZEOF_MINORNAME 12 +#define MODGETMINORNAME 13 +#define MODGETPATHLEN 14 +#define MODEVENTS 15 +#define MODGETFBNAME 16 +#define MODREREADDACF 17 +#define MODLOADDRVCONF 18 +#define MODUNLOADDRVCONF 19 +#define MODREMMAJBIND 20 +#define MODDEVT2INSTANCE 21 +#define MODGETDEVFSPATH_LEN 22 +#define MODGETDEVFSPATH 23 +#define MODDEVID2PATHS 24 +#define MODSETDEVPOLICY 26 +#define MODGETDEVPOLICY 27 +#define MODALLOCPRIV 28 +#define MODGETDEVPOLICYBYNAME 29 +#define MODLOADMINORPERM 31 +#define MODADDMINORPERM 32 +#define MODREMMINORPERM 33 +#define MODREMDRVCLEANUP 34 +#define MODDEVEXISTS 35 +#define MODDEVREADDIR 36 +#define MODDEVNAME 37 +#define MODGETDEVFSPATH_MI_LEN 38 +#define MODGETDEVFSPATH_MI 39 +#define MODRETIRE 40 +#define MODUNRETIRE 41 +#define MODISRETIRED 42 +#define MODDEVEMPTYDIR 43 +#define MODREMDRVALIAS 44 + +/* + * sub cmds for MODEVENTS + */ +#define MODEVENTS_FLUSH 0 +#define MODEVENTS_FLUSH_DUMP 1 +#define MODEVENTS_SET_DOOR_UPCALL_FILENAME 2 +#define MODEVENTS_GETDATA 3 +#define MODEVENTS_FREEDATA 4 +#define MODEVENTS_POST_EVENT 5 +#define MODEVENTS_REGISTER_EVENT 6 + +/* + * devname subcmds for MODDEVNAME + */ +#define MODDEVNAME_LOOKUPDOOR 0 +#define MODDEVNAME_DEVFSADMNODE 1 +#define MODDEVNAME_NSMAPS 2 +#define MODDEVNAME_PROFILE 3 +#define MODDEVNAME_RECONFIG 4 +#define MODDEVNAME_SYSAVAIL 5 + + +/* + * Data structure passed to modconfig command in kernel to build devfs tree + */ + +struct aliases { + struct aliases *a_next; + char *a_name; + int a_len; +}; + +#define MAXMODCONFNAME 256 + +struct modconfig { + char drvname[MAXMODCONFNAME]; + char drvclass[MAXMODCONFNAME]; + int major; + int flags; + int num_aliases; + struct aliases *ap; +}; + +#if defined(_SYSCALL32) + +struct aliases32 { + caddr32_t a_next; + caddr32_t a_name; + int32_t a_len; +}; + +struct modconfig32 { + char drvname[MAXMODCONFNAME]; + char drvclass[MAXMODCONFNAME]; + int32_t major; + int32_t flags; + int32_t num_aliases; + caddr32_t ap; +}; + +#endif /* _SYSCALL32 */ + +/* flags for modconfig */ +#define MOD_UNBIND_OVERRIDE 0x01 /* fail unbind if in use */ + +/* + * Max module path length + */ +#define MOD_MAXPATH 256 + +/* + * Default search path for modules ADDITIONAL to the directory + * where the kernel components we booted from are. + * + * Most often, this will be "/platform/{platform}/kernel /kernel /usr/kernel", + * but we don't wire it down here. + */ +#define MOD_DEFPATH "/kernel /usr/kernel" + +/* + * Default file name extension for autoloading modules. + */ +#define MOD_DEFEXT "" + +/* + * Parameters for modinfo + */ +#define MODMAXNAMELEN 32 /* max module name length */ +#define MODMAXLINKINFOLEN 32 /* max link info length */ + +/* + * Module specific information. + */ +struct modspecific_info { + char msi_linkinfo[MODMAXLINKINFOLEN]; /* name in linkage struct */ + int msi_p0; /* module specific information */ +}; + +/* + * Structure returned by modctl with MODINFO command. + */ +#define MODMAXLINK 10 /* max linkages modinfo can handle */ + +struct modinfo { + int mi_info; /* Flags for info wanted */ + int mi_state; /* Flags for module state */ + int mi_id; /* id of this loaded module */ + int mi_nextid; /* id of next module or -1 */ + caddr_t mi_base; /* virtual addr of text */ + size_t mi_size; /* size of module in bytes */ + int mi_rev; /* loadable modules rev */ + int mi_loadcnt; /* # of times loaded */ + char mi_name[MODMAXNAMELEN]; /* name of module */ + struct modspecific_info mi_msinfo[MODMAXLINK]; + /* mod specific info */ +}; + + +#if defined(_SYSCALL32) + +#define MODMAXNAMELEN32 32 /* max module name length */ +#define MODMAXLINKINFOLEN32 32 /* max link info length */ +#define MODMAXLINK32 10 /* max linkages modinfo can handle */ + +struct modspecific_info32 { + char msi_linkinfo[MODMAXLINKINFOLEN32]; /* name in linkage struct */ + int32_t msi_p0; /* module specific information */ +}; + +struct modinfo32 { + int32_t mi_info; /* Flags for info wanted */ + int32_t mi_state; /* Flags for module state */ + int32_t mi_id; /* id of this loaded module */ + int32_t mi_nextid; /* id of next module or -1 */ + caddr32_t mi_base; /* virtual addr of text */ + uint32_t mi_size; /* size of module in bytes */ + int32_t mi_rev; /* loadable modules rev */ + int32_t mi_loadcnt; /* # of times loaded */ + char mi_name[MODMAXNAMELEN32]; /* name of module */ + struct modspecific_info32 mi_msinfo[MODMAXLINK32]; + /* mod specific info */ +}; + +#endif /* _SYSCALL32 */ + +/* Values for mi_info flags */ +#define MI_INFO_ONE 1 +#define MI_INFO_ALL 2 +#define MI_INFO_CNT 4 +#define MI_INFO_LINKAGE 8 /* used internally to extract modlinkage */ +/* + * MI_INFO_NOBASE indicates caller does not need mi_base. Failure to use this + * flag may lead 32-bit apps to receive an EOVERFLOW error from modctl(MODINFO) + * when used with a 64-bit kernel. + */ +#define MI_INFO_NOBASE 16 + +/* Values for mi_state */ +#define MI_LOADED 1 +#define MI_INSTALLED 2 + +/* + * Macros to vector to the appropriate module specific routine. + */ +#define MODL_INSTALL(MODL, MODLP) \ + (*(MODL)->misc_modops->modm_install)(MODL, MODLP) +#define MODL_REMOVE(MODL, MODLP) \ + (*(MODL)->misc_modops->modm_remove)(MODL, MODLP) +#define MODL_INFO(MODL, MODLP, P0) \ + (*(MODL)->misc_modops->modm_info)(MODL, MODLP, P0) + +/* + * Definitions for stubs + */ +struct mod_stub_info { + uintptr_t mods_func_adr; + struct mod_modinfo *mods_modinfo; + uintptr_t mods_stub_adr; + int (*mods_errfcn)(void); + int mods_flag; /* flags defined below */ +}; + +/* + * Definitions for mods_flag. + */ +#define MODS_WEAK 0x01 /* weak stub (not loaded if called) */ +#define MODS_NOUNLOAD 0x02 /* module not unloadable (no _fini()) */ +#define MODS_INSTALLED 0x10 /* module installed */ + +struct mod_modinfo { + char *modm_module_name; + struct modctl *mp; + struct mod_stub_info modm_stubs[1]; +}; + +struct modctl_list { + struct modctl_list *modl_next; + struct modctl *modl_modp; +}; + +/* + * Structure to manage a loadable module. + * Note: the module (mod_mp) structure's "text" and "text_size" information + * are replicated in the modctl structure so that mod_containing_pc() + * doesn't have to grab any locks (modctls are persistent; modules are not.) + */ +typedef struct modctl { + struct modctl *mod_next; /* &modules based list */ + struct modctl *mod_prev; + int mod_id; + void *mod_mp; + kthread_t *mod_inprogress_thread; + struct mod_modinfo *mod_modinfo; + struct modlinkage *mod_linkage; + char *mod_filename; + char *mod_modname; + + char mod_busy; /* inprogress_thread has locked */ + char mod_want; /* someone waiting for unlock */ + char mod_prim; /* primary module */ + + int mod_ref; /* ref count - from dependent or stub */ + + char mod_loaded; /* module in memory */ + char mod_installed; /* post _init pre _fini */ + char mod_loadflags; + char mod_delay_unload; /* deferred unload */ + + struct modctl_list *mod_requisites; /* mods this one depends on. */ + void *__unused; /* NOTE: reuse (same size) is OK, */ + /* deletion causes mdb.vs.core issues */ + int mod_loadcnt; /* number of times mod was loaded */ + int mod_nenabled; /* # of enabled DTrace probes in mod */ + char *mod_text; + size_t mod_text_size; + + int mod_gencount; /* # times loaded/unloaded */ + struct modctl *mod_requisite_loading; /* mod circular dependency */ +} modctl_t; + +/* + * mod_loadflags + */ + +#define MOD_NOAUTOUNLOAD 0x1 /* Auto mod-unloader skips this mod */ +#define MOD_NONOTIFY 0x2 /* No krtld notifications on (un)load */ +#define MOD_NOUNLOAD 0x4 /* Assume EBUSY for all _fini's */ + +#define MOD_BIND_HASHSIZE 64 +#define MOD_BIND_HASHMASK (MOD_BIND_HASHSIZE-1) + +typedef int modid_t; + +/* + * global function and data declarations + */ +extern kmutex_t mod_lock; + +extern char *systemfile; +extern char **syscallnames; +extern int moddebug; + +/* + * this is the head of a doubly linked list. Only the next and prev + * pointers are used + */ +extern modctl_t modules; + +/* + * Only the following are part of the DDI/DKI + */ +extern int mod_install(struct modlinkage *); +extern int mod_remove(struct modlinkage *); +extern int mod_info(struct modlinkage *, struct modinfo *); + +/* + * bit definitions for moddebug. + */ +#define MODDEBUG_LOADMSG 0x80000000 /* print "[un]loading..." msg */ +#define MODDEBUG_ERRMSG 0x40000000 /* print detailed error msgs */ +#define MODDEBUG_LOADMSG2 0x20000000 /* print 2nd level msgs */ +#define MODDEBUG_RETIRE 0x10000000 /* print retire msgs */ +#define MODDEBUG_BINDING 0x00040000 /* driver/alias binding */ +#define MODDEBUG_FINI_EBUSY 0x00020000 /* pretend fini returns EBUSY */ +#define MODDEBUG_NOAUL_IPP 0x00010000 /* no Autounloading ipp mods */ +#define MODDEBUG_NOAUL_DACF 0x00008000 /* no Autounloading dacf mods */ +#define MODDEBUG_KEEPTEXT 0x00004000 /* keep text after unloading */ +#define MODDEBUG_NOAUL_DRV 0x00001000 /* no Autounloading Drivers */ +#define MODDEBUG_NOAUL_EXEC 0x00000800 /* no Autounloading Execs */ +#define MODDEBUG_NOAUL_FS 0x00000400 /* no Autounloading File sys */ +#define MODDEBUG_NOAUL_MISC 0x00000200 /* no Autounloading misc */ +#define MODDEBUG_NOAUL_SCHED 0x00000100 /* no Autounloading scheds */ +#define MODDEBUG_NOAUL_STR 0x00000080 /* no Autounloading streams */ +#define MODDEBUG_NOAUL_SYS 0x00000040 /* no Autounloading syscalls */ +#define MODDEBUG_NOCTF 0x00000020 /* do not load CTF debug data */ +#define MODDEBUG_NOAUTOUNLOAD 0x00000010 /* no autounloading at all */ +#define MODDEBUG_DDI_MOD 0x00000008 /* ddi_mod{open,sym,close} */ +#define MODDEBUG_MP_MATCH 0x00000004 /* dev_minorperm */ +#define MODDEBUG_MINORPERM 0x00000002 /* minor perm modctls */ +#define MODDEBUG_USERDEBUG 0x00000001 /* bpt after init_module() */ + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_MODCTL_H */ diff --git a/module/icp/include/sys/modhash.h b/module/icp/include/sys/modhash.h new file mode 100644 index 000000000000..06b52ff02604 --- /dev/null +++ b/module/icp/include/sys/modhash.h @@ -0,0 +1,147 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_MODHASH_H +#define _SYS_MODHASH_H + +/* + * Generic hash implementation for the kernel. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +/* + * Opaque data types for storing keys and values + */ +typedef void *mod_hash_val_t; +typedef void *mod_hash_key_t; + +/* + * Opaque data type for reservation + */ +typedef void *mod_hash_hndl_t; + +/* + * Opaque type for hash itself. + */ +struct mod_hash; +typedef struct mod_hash mod_hash_t; + +/* + * String hash table + */ +mod_hash_t *mod_hash_create_strhash_nodtr(char *, size_t, + void (*)(mod_hash_val_t)); +mod_hash_t *mod_hash_create_strhash(char *, size_t, void (*)(mod_hash_val_t)); +void mod_hash_destroy_strhash(mod_hash_t *); +int mod_hash_strkey_cmp(mod_hash_key_t, mod_hash_key_t); +void mod_hash_strkey_dtor(mod_hash_key_t); +void mod_hash_strval_dtor(mod_hash_val_t); +uint_t mod_hash_bystr(void *, mod_hash_key_t); + +/* + * Pointer hash table + */ +mod_hash_t *mod_hash_create_ptrhash(char *, size_t, void (*)(mod_hash_val_t), + size_t); +void mod_hash_destroy_ptrhash(mod_hash_t *); +int mod_hash_ptrkey_cmp(mod_hash_key_t, mod_hash_key_t); +uint_t mod_hash_byptr(void *, mod_hash_key_t); + +/* + * ID hash table + */ +mod_hash_t *mod_hash_create_idhash(char *, size_t, void (*)(mod_hash_val_t)); +void mod_hash_destroy_idhash(mod_hash_t *); +int mod_hash_idkey_cmp(mod_hash_key_t, mod_hash_key_t); +uint_t mod_hash_byid(void *, mod_hash_key_t); +uint_t mod_hash_iddata_gen(size_t); + +/* + * Hash management functions + */ +mod_hash_t *mod_hash_create_extended(char *, size_t, void (*)(mod_hash_key_t), + void (*)(mod_hash_val_t), uint_t (*)(void *, mod_hash_key_t), void *, + int (*)(mod_hash_key_t, mod_hash_key_t), int); + +void mod_hash_destroy_hash(mod_hash_t *); +void mod_hash_clear(mod_hash_t *); + +/* + * Null key and value destructors + */ +void mod_hash_null_keydtor(mod_hash_key_t); +void mod_hash_null_valdtor(mod_hash_val_t); + +/* + * Basic hash operations + */ + +/* + * Error codes for insert, remove, find, destroy. + */ +#define MH_ERR_NOMEM -1 +#define MH_ERR_DUPLICATE -2 +#define MH_ERR_NOTFOUND -3 + +/* + * Return codes for hash walkers + */ +#define MH_WALK_CONTINUE 0 +#define MH_WALK_TERMINATE 1 + +/* + * Basic hash operations + */ +int mod_hash_insert(mod_hash_t *, mod_hash_key_t, mod_hash_val_t); +int mod_hash_replace(mod_hash_t *, mod_hash_key_t, mod_hash_val_t); +int mod_hash_remove(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *); +int mod_hash_destroy(mod_hash_t *, mod_hash_key_t); +int mod_hash_find(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *); +int mod_hash_find_cb(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *, + void (*)(mod_hash_key_t, mod_hash_val_t)); +int mod_hash_find_cb_rval(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *, + int (*)(mod_hash_key_t, mod_hash_val_t), int *); +void mod_hash_walk(mod_hash_t *, + uint_t (*)(mod_hash_key_t, mod_hash_val_t *, void *), void *); + +/* + * Reserving hash operations + */ +int mod_hash_reserve(mod_hash_t *, mod_hash_hndl_t *); +int mod_hash_reserve_nosleep(mod_hash_t *, mod_hash_hndl_t *); +void mod_hash_cancel(mod_hash_t *, mod_hash_hndl_t *); +int mod_hash_insert_reserve(mod_hash_t *, mod_hash_key_t, mod_hash_val_t, + mod_hash_hndl_t); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_MODHASH_H */ diff --git a/module/icp/include/sys/modhash_impl.h b/module/icp/include/sys/modhash_impl.h new file mode 100644 index 000000000000..3130773aa196 --- /dev/null +++ b/module/icp/include/sys/modhash_impl.h @@ -0,0 +1,108 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_MODHASH_IMPL_H +#define _SYS_MODHASH_IMPL_H + +/* + * Internal details for the kernel's generic hash implementation. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +struct mod_hash_entry { + mod_hash_key_t mhe_key; /* stored hash key */ + mod_hash_val_t mhe_val; /* stored hash value */ + struct mod_hash_entry *mhe_next; /* next item in chain */ +}; + +struct mod_hash_stat { + ulong_t mhs_hit; /* tried a 'find' and it succeeded */ + ulong_t mhs_miss; /* tried a 'find' but it failed */ + ulong_t mhs_coll; /* occur when insert fails because of dup's */ + ulong_t mhs_nelems; /* total number of stored key/value pairs */ + ulong_t mhs_nomem; /* number of times kmem_alloc failed */ +}; + +struct mod_hash { + krwlock_t mh_contents; /* lock protecting contents */ + char *mh_name; /* hash name */ + int mh_sleep; /* kmem_alloc flag */ + size_t mh_nchains; /* # of elements in mh_entries */ + + /* key and val destructor */ + void (*mh_kdtor)(mod_hash_key_t); + void (*mh_vdtor)(mod_hash_val_t); + + /* key comparator */ + int (*mh_keycmp)(mod_hash_key_t, mod_hash_key_t); + + /* hash algorithm, and algorithm-private data */ + uint_t (*mh_hashalg)(void *, mod_hash_key_t); + void *mh_hashalg_data; + + struct mod_hash *mh_next; /* next hash in list */ + + struct mod_hash_stat mh_stat; + + struct mod_hash_entry *mh_entries[1]; +}; + +/* + * MH_SIZE() + * Compute the size of a mod_hash_t, in bytes, given the number of + * elements it contains. + */ +#define MH_SIZE(n) \ + (sizeof (mod_hash_t) + ((n) - 1) * (sizeof (struct mod_hash_entry *))) + +/* + * Module initialization; called once. + */ +void mod_hash_fini(void); +void mod_hash_init(void); + +/* + * Internal routines. Use directly with care. + */ +uint_t i_mod_hash(mod_hash_t *, mod_hash_key_t); +int i_mod_hash_insert_nosync(mod_hash_t *, mod_hash_key_t, mod_hash_val_t, + mod_hash_hndl_t); +int i_mod_hash_remove_nosync(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *); +int i_mod_hash_find_nosync(mod_hash_t *, mod_hash_key_t, mod_hash_val_t *); +void i_mod_hash_walk_nosync(mod_hash_t *, uint_t (*)(mod_hash_key_t, + mod_hash_val_t *, void *), void *); +void i_mod_hash_clear_nosync(mod_hash_t *hash); + +#ifdef __cplusplus +} +#endif + +#endif /* _SYS_MODHASH_IMPL_H */ diff --git a/module/icp/include/sys/stack.h b/module/icp/include/sys/stack.h new file mode 100644 index 000000000000..64fecf409b5c --- /dev/null +++ b/module/icp/include/sys/stack.h @@ -0,0 +1,36 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_STACK_H +#define _SYS_STACK_H + +#if defined(__i386) || defined(__amd64) + +#include /* XX64 x86/sys/stack.h */ + +#endif + +#endif /* _SYS_STACK_H */ diff --git a/module/icp/include/sys/trap.h b/module/icp/include/sys/trap.h new file mode 100644 index 000000000000..7f9fd375805f --- /dev/null +++ b/module/icp/include/sys/trap.h @@ -0,0 +1,36 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2005 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _SYS_TRAP_H +#define _SYS_TRAP_H + +#if defined(__i386) || defined(__amd64) + +#include /* XX64 x86/sys/trap.h */ + +#endif + +#endif /* _SYS_TRAP_H */ diff --git a/module/icp/io/aes.c b/module/icp/io/aes.c new file mode 100644 index 000000000000..ada697eb6d52 --- /dev/null +++ b/module/icp/io/aes.c @@ -0,0 +1,1437 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + */ + +/* + * AES provider for the Kernel Cryptographic Framework (KCF) + */ + +#include +#include +#include +#include +#include +#include +#include +#define _AES_IMPL +#include + +#define CRYPTO_PROVIDER_NAME "aes" + +extern struct mod_ops mod_cryptoops; + +/* + * Module linkage information for the kernel. + */ +static struct modlcrypto modlcrypto = { + &mod_cryptoops, + "AES Kernel SW Provider" +}; + +static struct modlinkage modlinkage = { + MODREV_1, { (void *)&modlcrypto, NULL } +}; + +/* + * Mechanism info structure passed to KCF during registration. + */ +static crypto_mech_info_t aes_mech_info_tab[] = { + /* AES_ECB */ + {SUN_CKM_AES_ECB, AES_ECB_MECH_INFO_TYPE, + CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | + CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, + AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, + /* AES_CBC */ + {SUN_CKM_AES_CBC, AES_CBC_MECH_INFO_TYPE, + CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | + CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, + AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, + /* AES_CTR */ + {SUN_CKM_AES_CTR, AES_CTR_MECH_INFO_TYPE, + CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | + CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, + AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, + /* AES_CCM */ + {SUN_CKM_AES_CCM, AES_CCM_MECH_INFO_TYPE, + CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | + CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, + AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, + /* AES_GCM */ + {SUN_CKM_AES_GCM, AES_GCM_MECH_INFO_TYPE, + CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | + CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, + AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, + /* AES_GMAC */ + {SUN_CKM_AES_GMAC, AES_GMAC_MECH_INFO_TYPE, + CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | + CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC | + CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC | + CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC | + CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC, + AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES} +}; + +/* operations are in-place if the output buffer is NULL */ +#define AES_ARG_INPLACE(input, output) \ + if ((output) == NULL) \ + (output) = (input); + +static void aes_provider_status(crypto_provider_handle_t, uint_t *); + +static crypto_control_ops_t aes_control_ops = { + aes_provider_status +}; + +static int aes_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *, + crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); +static int aes_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *, + crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); +static int aes_common_init(crypto_ctx_t *, crypto_mechanism_t *, + crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t, boolean_t); +static int aes_common_init_ctx(aes_ctx_t *, crypto_spi_ctx_template_t *, + crypto_mechanism_t *, crypto_key_t *, int, boolean_t); +static int aes_encrypt_final(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); +static int aes_decrypt_final(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); + +static int aes_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, + crypto_req_handle_t); +static int aes_encrypt_update(crypto_ctx_t *, crypto_data_t *, + crypto_data_t *, crypto_req_handle_t); +static int aes_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); + +static int aes_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, + crypto_req_handle_t); +static int aes_decrypt_update(crypto_ctx_t *, crypto_data_t *, + crypto_data_t *, crypto_req_handle_t); +static int aes_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, + crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); + +static crypto_cipher_ops_t aes_cipher_ops = { + aes_encrypt_init, + aes_encrypt, + aes_encrypt_update, + aes_encrypt_final, + aes_encrypt_atomic, + aes_decrypt_init, + aes_decrypt, + aes_decrypt_update, + aes_decrypt_final, + aes_decrypt_atomic +}; + +static int aes_mac_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); +static int aes_mac_verify_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); + +static crypto_mac_ops_t aes_mac_ops = { + NULL, + NULL, + NULL, + NULL, + aes_mac_atomic, + aes_mac_verify_atomic +}; + +static int aes_create_ctx_template(crypto_provider_handle_t, + crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *, + size_t *, crypto_req_handle_t); +static int aes_free_context(crypto_ctx_t *); + +static crypto_ctx_ops_t aes_ctx_ops = { + aes_create_ctx_template, + aes_free_context +}; + +static crypto_ops_t aes_crypto_ops = {{{{{ + &aes_control_ops, + NULL, + &aes_cipher_ops, + &aes_mac_ops, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + &aes_ctx_ops +}}}}}; + +static crypto_provider_info_t aes_prov_info = {{{{ + CRYPTO_SPI_VERSION_1, + "AES Software Provider", + CRYPTO_SW_PROVIDER, + NULL, + &aes_crypto_ops, + sizeof (aes_mech_info_tab)/sizeof (crypto_mech_info_t), + aes_mech_info_tab +}}}}; + +static crypto_kcf_provider_handle_t aes_prov_handle = 0; +static crypto_data_t null_crypto_data = { CRYPTO_DATA_RAW }; + +int +aes_mod_init(void) +{ + int ret; + + if ((ret = mod_install(&modlinkage)) != 0) + return (ret); + + /* Register with KCF. If the registration fails, remove the module. */ + if (crypto_register_provider(&aes_prov_info, &aes_prov_handle)) { + (void) mod_remove(&modlinkage); + return (EACCES); + } + + return (0); +} + +int +aes_mod_fini(void) +{ + /* Unregister from KCF if module is registered */ + if (aes_prov_handle != 0) { + if (crypto_unregister_provider(aes_prov_handle)) + return (EBUSY); + + aes_prov_handle = 0; + } + + return (mod_remove(&modlinkage)); +} + +static int +aes_check_mech_param(crypto_mechanism_t *mechanism, aes_ctx_t **ctx, int kmflag) +{ + void *p = NULL; + boolean_t param_required = B_TRUE; + size_t param_len; + void *(*alloc_fun)(int); + int rv = CRYPTO_SUCCESS; + + switch (mechanism->cm_type) { + case AES_ECB_MECH_INFO_TYPE: + param_required = B_FALSE; + alloc_fun = ecb_alloc_ctx; + break; + case AES_CBC_MECH_INFO_TYPE: + param_len = AES_BLOCK_LEN; + alloc_fun = cbc_alloc_ctx; + break; + case AES_CTR_MECH_INFO_TYPE: + param_len = sizeof (CK_AES_CTR_PARAMS); + alloc_fun = ctr_alloc_ctx; + break; + case AES_CCM_MECH_INFO_TYPE: + param_len = sizeof (CK_AES_CCM_PARAMS); + alloc_fun = ccm_alloc_ctx; + break; + case AES_GCM_MECH_INFO_TYPE: + param_len = sizeof (CK_AES_GCM_PARAMS); + alloc_fun = gcm_alloc_ctx; + break; + case AES_GMAC_MECH_INFO_TYPE: + param_len = sizeof (CK_AES_GMAC_PARAMS); + alloc_fun = gmac_alloc_ctx; + break; + default: + rv = CRYPTO_MECHANISM_INVALID; + return (rv); + } + if (param_required && mechanism->cm_param != NULL && + mechanism->cm_param_len != param_len) { + rv = CRYPTO_MECHANISM_PARAM_INVALID; + } + if (ctx != NULL) { + p = (alloc_fun)(kmflag); + *ctx = p; + } + return (rv); +} + +/* + * Initialize key schedules for AES + */ +static int +init_keysched(crypto_key_t *key, void *newbie) +{ + /* + * Only keys by value are supported by this module. + */ + switch (key->ck_format) { + case CRYPTO_KEY_RAW: + if (key->ck_length < AES_MINBITS || + key->ck_length > AES_MAXBITS) { + return (CRYPTO_KEY_SIZE_RANGE); + } + + /* key length must be either 128, 192, or 256 */ + if ((key->ck_length & 63) != 0) + return (CRYPTO_KEY_SIZE_RANGE); + break; + default: + return (CRYPTO_KEY_TYPE_INCONSISTENT); + } + + aes_init_keysched(key->ck_data, key->ck_length, newbie); + return (CRYPTO_SUCCESS); +} + +/* + * KCF software provider control entry points. + */ +/* ARGSUSED */ +static void +aes_provider_status(crypto_provider_handle_t provider, uint_t *status) +{ + *status = CRYPTO_PROVIDER_READY; +} + +static int +aes_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_spi_ctx_template_t template, + crypto_req_handle_t req) { + return (aes_common_init(ctx, mechanism, key, template, req, B_TRUE)); +} + +static int +aes_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_spi_ctx_template_t template, + crypto_req_handle_t req) { + return (aes_common_init(ctx, mechanism, key, template, req, B_FALSE)); +} + + + +/* + * KCF software provider encrypt entry points. + */ +static int +aes_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_spi_ctx_template_t template, + crypto_req_handle_t req, boolean_t is_encrypt_init) +{ + aes_ctx_t *aes_ctx; + int rv; + int kmflag; + + /* + * Only keys by value are supported by this module. + */ + if (key->ck_format != CRYPTO_KEY_RAW) { + return (CRYPTO_KEY_TYPE_INCONSISTENT); + } + + kmflag = crypto_kmflag(req); + if ((rv = aes_check_mech_param(mechanism, &aes_ctx, kmflag)) + != CRYPTO_SUCCESS) + return (rv); + + rv = aes_common_init_ctx(aes_ctx, template, mechanism, key, kmflag, + is_encrypt_init); + if (rv != CRYPTO_SUCCESS) { + crypto_free_mode_ctx(aes_ctx); + return (rv); + } + + ctx->cc_provider_private = aes_ctx; + + return (CRYPTO_SUCCESS); +} + +static void +aes_copy_block64(uint8_t *in, uint64_t *out) +{ + if (IS_P2ALIGNED(in, sizeof (uint64_t))) { + /* LINTED: pointer alignment */ + out[0] = *(uint64_t *)&in[0]; + /* LINTED: pointer alignment */ + out[1] = *(uint64_t *)&in[8]; + } else { + uint8_t *iv8 = (uint8_t *)&out[0]; + + AES_COPY_BLOCK(in, iv8); + } +} + + +static int +aes_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext, + crypto_data_t *ciphertext, crypto_req_handle_t req) +{ + int ret = CRYPTO_FAILED; + + aes_ctx_t *aes_ctx; + size_t saved_length, saved_offset, length_needed; + + ASSERT(ctx->cc_provider_private != NULL); + aes_ctx = ctx->cc_provider_private; + + /* + * For block ciphers, plaintext must be a multiple of AES block size. + * This test is only valid for ciphers whose blocksize is a power of 2. + */ + if (((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) + == 0) && (plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0) + return (CRYPTO_DATA_LEN_RANGE); + + AES_ARG_INPLACE(plaintext, ciphertext); + + /* + * We need to just return the length needed to store the output. + * We should not destroy the context for the following case. + */ + switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) { + case CCM_MODE: + length_needed = plaintext->cd_length + aes_ctx->ac_mac_len; + break; + case GCM_MODE: + length_needed = plaintext->cd_length + aes_ctx->ac_tag_len; + break; + case GMAC_MODE: + if (plaintext->cd_length != 0) + return (CRYPTO_ARGUMENTS_BAD); + + length_needed = aes_ctx->ac_tag_len; + break; + default: + length_needed = plaintext->cd_length; + } + + if (ciphertext->cd_length < length_needed) { + ciphertext->cd_length = length_needed; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + saved_length = ciphertext->cd_length; + saved_offset = ciphertext->cd_offset; + + /* + * Do an update on the specified input data. + */ + ret = aes_encrypt_update(ctx, plaintext, ciphertext, req); + if (ret != CRYPTO_SUCCESS) { + return (ret); + } + + /* + * For CCM mode, aes_ccm_encrypt_final() will take care of any + * left-over unprocessed data, and compute the MAC + */ + if (aes_ctx->ac_flags & CCM_MODE) { + /* + * ccm_encrypt_final() will compute the MAC and append + * it to existing ciphertext. So, need to adjust the left over + * length value accordingly + */ + + /* order of following 2 lines MUST not be reversed */ + ciphertext->cd_offset = ciphertext->cd_length; + ciphertext->cd_length = saved_length - ciphertext->cd_length; + ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext, + AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); + if (ret != CRYPTO_SUCCESS) { + return (ret); + } + + if (plaintext != ciphertext) { + ciphertext->cd_length = + ciphertext->cd_offset - saved_offset; + } + ciphertext->cd_offset = saved_offset; + } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + /* + * gcm_encrypt_final() will compute the MAC and append + * it to existing ciphertext. So, need to adjust the left over + * length value accordingly + */ + + /* order of following 2 lines MUST not be reversed */ + ciphertext->cd_offset = ciphertext->cd_length; + ciphertext->cd_length = saved_length - ciphertext->cd_length; + ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext, + AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + if (ret != CRYPTO_SUCCESS) { + return (ret); + } + + if (plaintext != ciphertext) { + ciphertext->cd_length = + ciphertext->cd_offset - saved_offset; + } + ciphertext->cd_offset = saved_offset; + } + + ASSERT(aes_ctx->ac_remainder_len == 0); + (void) aes_free_context(ctx); + + return (ret); +} + + +static int +aes_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext, + crypto_data_t *plaintext, crypto_req_handle_t req) +{ + int ret = CRYPTO_FAILED; + + aes_ctx_t *aes_ctx; + off_t saved_offset; + size_t saved_length, length_needed; + + ASSERT(ctx->cc_provider_private != NULL); + aes_ctx = ctx->cc_provider_private; + + /* + * For block ciphers, plaintext must be a multiple of AES block size. + * This test is only valid for ciphers whose blocksize is a power of 2. + */ + if (((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) + == 0) && (ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0) { + return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); + } + + AES_ARG_INPLACE(ciphertext, plaintext); + + /* + * Return length needed to store the output. + * Do not destroy context when plaintext buffer is too small. + * + * CCM: plaintext is MAC len smaller than cipher text + * GCM: plaintext is TAG len smaller than cipher text + * GMAC: plaintext length must be zero + */ + switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) { + case CCM_MODE: + length_needed = aes_ctx->ac_processed_data_len; + break; + case GCM_MODE: + length_needed = ciphertext->cd_length - aes_ctx->ac_tag_len; + break; + case GMAC_MODE: + if (plaintext->cd_length != 0) + return (CRYPTO_ARGUMENTS_BAD); + + length_needed = 0; + break; + default: + length_needed = ciphertext->cd_length; + } + + if (plaintext->cd_length < length_needed) { + plaintext->cd_length = length_needed; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + saved_offset = plaintext->cd_offset; + saved_length = plaintext->cd_length; + + /* + * Do an update on the specified input data. + */ + ret = aes_decrypt_update(ctx, ciphertext, plaintext, req); + if (ret != CRYPTO_SUCCESS) { + goto cleanup; + } + + if (aes_ctx->ac_flags & CCM_MODE) { + ASSERT(aes_ctx->ac_processed_data_len == aes_ctx->ac_data_len); + ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len); + + /* order of following 2 lines MUST not be reversed */ + plaintext->cd_offset = plaintext->cd_length; + plaintext->cd_length = saved_length - plaintext->cd_length; + + ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext, + AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + if (ret == CRYPTO_SUCCESS) { + if (plaintext != ciphertext) { + plaintext->cd_length = + plaintext->cd_offset - saved_offset; + } + } else { + plaintext->cd_length = saved_length; + } + + plaintext->cd_offset = saved_offset; + } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + /* order of following 2 lines MUST not be reversed */ + plaintext->cd_offset = plaintext->cd_length; + plaintext->cd_length = saved_length - plaintext->cd_length; + + ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext, + AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); + if (ret == CRYPTO_SUCCESS) { + if (plaintext != ciphertext) { + plaintext->cd_length = + plaintext->cd_offset - saved_offset; + } + } else { + plaintext->cd_length = saved_length; + } + + plaintext->cd_offset = saved_offset; + } + + ASSERT(aes_ctx->ac_remainder_len == 0); + +cleanup: + (void) aes_free_context(ctx); + + return (ret); +} + + +/* ARGSUSED */ +static int +aes_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext, + crypto_data_t *ciphertext, crypto_req_handle_t req) +{ + off_t saved_offset; + size_t saved_length, out_len; + int ret = CRYPTO_SUCCESS; + aes_ctx_t *aes_ctx; + + ASSERT(ctx->cc_provider_private != NULL); + aes_ctx = ctx->cc_provider_private; + + AES_ARG_INPLACE(plaintext, ciphertext); + + /* compute number of bytes that will hold the ciphertext */ + out_len = aes_ctx->ac_remainder_len; + out_len += plaintext->cd_length; + out_len &= ~(AES_BLOCK_LEN - 1); + + /* return length needed to store the output */ + if (ciphertext->cd_length < out_len) { + ciphertext->cd_length = out_len; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + saved_offset = ciphertext->cd_offset; + saved_length = ciphertext->cd_length; + + /* + * Do the AES update on the specified input data. + */ + switch (plaintext->cd_format) { + case CRYPTO_DATA_RAW: + ret = crypto_update_iov(ctx->cc_provider_private, + plaintext, ciphertext, aes_encrypt_contiguous_blocks, + aes_copy_block64); + break; + case CRYPTO_DATA_UIO: + ret = crypto_update_uio(ctx->cc_provider_private, + plaintext, ciphertext, aes_encrypt_contiguous_blocks, + aes_copy_block64); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + /* + * Since AES counter mode is a stream cipher, we call + * ctr_mode_final() to pick up any remaining bytes. + * It is an internal function that does not destroy + * the context like *normal* final routines. + */ + if ((aes_ctx->ac_flags & CTR_MODE) && (aes_ctx->ac_remainder_len > 0)) { + ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, + ciphertext, aes_encrypt_block); + } + + if (ret == CRYPTO_SUCCESS) { + if (plaintext != ciphertext) + ciphertext->cd_length = + ciphertext->cd_offset - saved_offset; + } else { + ciphertext->cd_length = saved_length; + } + ciphertext->cd_offset = saved_offset; + + return (ret); +} + + +static int +aes_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext, + crypto_data_t *plaintext, crypto_req_handle_t req) +{ + off_t saved_offset; + size_t saved_length, out_len; + int ret = CRYPTO_SUCCESS; + aes_ctx_t *aes_ctx; + + ASSERT(ctx->cc_provider_private != NULL); + aes_ctx = ctx->cc_provider_private; + + AES_ARG_INPLACE(ciphertext, plaintext); + + /* + * Compute number of bytes that will hold the plaintext. + * This is not necessary for CCM, GCM, and GMAC since these + * mechanisms never return plaintext for update operations. + */ + if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) { + out_len = aes_ctx->ac_remainder_len; + out_len += ciphertext->cd_length; + out_len &= ~(AES_BLOCK_LEN - 1); + + /* return length needed to store the output */ + if (plaintext->cd_length < out_len) { + plaintext->cd_length = out_len; + return (CRYPTO_BUFFER_TOO_SMALL); + } + } + + saved_offset = plaintext->cd_offset; + saved_length = plaintext->cd_length; + + if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) + gcm_set_kmflag((gcm_ctx_t *)aes_ctx, crypto_kmflag(req)); + + /* + * Do the AES update on the specified input data. + */ + switch (ciphertext->cd_format) { + case CRYPTO_DATA_RAW: + ret = crypto_update_iov(ctx->cc_provider_private, + ciphertext, plaintext, aes_decrypt_contiguous_blocks, + aes_copy_block64); + break; + case CRYPTO_DATA_UIO: + ret = crypto_update_uio(ctx->cc_provider_private, + ciphertext, plaintext, aes_decrypt_contiguous_blocks, + aes_copy_block64); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + /* + * Since AES counter mode is a stream cipher, we call + * ctr_mode_final() to pick up any remaining bytes. + * It is an internal function that does not destroy + * the context like *normal* final routines. + */ + if ((aes_ctx->ac_flags & CTR_MODE) && (aes_ctx->ac_remainder_len > 0)) { + ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, plaintext, + aes_encrypt_block); + if (ret == CRYPTO_DATA_LEN_RANGE) + ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; + } + + if (ret == CRYPTO_SUCCESS) { + if (ciphertext != plaintext) + plaintext->cd_length = + plaintext->cd_offset - saved_offset; + } else { + plaintext->cd_length = saved_length; + } + plaintext->cd_offset = saved_offset; + + + return (ret); +} + +/* ARGSUSED */ +static int +aes_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *data, + crypto_req_handle_t req) +{ + aes_ctx_t *aes_ctx; + int ret; + + ASSERT(ctx->cc_provider_private != NULL); + aes_ctx = ctx->cc_provider_private; + + if (data->cd_format != CRYPTO_DATA_RAW && + data->cd_format != CRYPTO_DATA_UIO) { + return (CRYPTO_ARGUMENTS_BAD); + } + + if (aes_ctx->ac_flags & CTR_MODE) { + if (aes_ctx->ac_remainder_len > 0) { + ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, data, + aes_encrypt_block); + if (ret != CRYPTO_SUCCESS) + return (ret); + } + } else if (aes_ctx->ac_flags & CCM_MODE) { + ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data, + AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); + if (ret != CRYPTO_SUCCESS) { + return (ret); + } + } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + size_t saved_offset = data->cd_offset; + + ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data, + AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + if (ret != CRYPTO_SUCCESS) { + return (ret); + } + data->cd_length = data->cd_offset - saved_offset; + data->cd_offset = saved_offset; + } else { + /* + * There must be no unprocessed plaintext. + * This happens if the length of the last data is + * not a multiple of the AES block length. + */ + if (aes_ctx->ac_remainder_len > 0) { + return (CRYPTO_DATA_LEN_RANGE); + } + data->cd_length = 0; + } + + (void) aes_free_context(ctx); + + return (CRYPTO_SUCCESS); +} + +/* ARGSUSED */ +static int +aes_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *data, + crypto_req_handle_t req) +{ + aes_ctx_t *aes_ctx; + int ret; + off_t saved_offset; + size_t saved_length; + + ASSERT(ctx->cc_provider_private != NULL); + aes_ctx = ctx->cc_provider_private; + + if (data->cd_format != CRYPTO_DATA_RAW && + data->cd_format != CRYPTO_DATA_UIO) { + return (CRYPTO_ARGUMENTS_BAD); + } + + /* + * There must be no unprocessed ciphertext. + * This happens if the length of the last ciphertext is + * not a multiple of the AES block length. + */ + if (aes_ctx->ac_remainder_len > 0) { + if ((aes_ctx->ac_flags & CTR_MODE) == 0) + return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); + else { + ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, data, + aes_encrypt_block); + if (ret == CRYPTO_DATA_LEN_RANGE) + ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; + if (ret != CRYPTO_SUCCESS) + return (ret); + } + } + + if (aes_ctx->ac_flags & CCM_MODE) { + /* + * This is where all the plaintext is returned, make sure + * the plaintext buffer is big enough + */ + size_t pt_len = aes_ctx->ac_data_len; + if (data->cd_length < pt_len) { + data->cd_length = pt_len; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + ASSERT(aes_ctx->ac_processed_data_len == pt_len); + ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len); + saved_offset = data->cd_offset; + saved_length = data->cd_length; + ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data, + AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + if (ret == CRYPTO_SUCCESS) { + data->cd_length = data->cd_offset - saved_offset; + } else { + data->cd_length = saved_length; + } + + data->cd_offset = saved_offset; + if (ret != CRYPTO_SUCCESS) { + return (ret); + } + } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { + /* + * This is where all the plaintext is returned, make sure + * the plaintext buffer is big enough + */ + gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx; + size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len; + + if (data->cd_length < pt_len) { + data->cd_length = pt_len; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + saved_offset = data->cd_offset; + saved_length = data->cd_length; + ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data, + AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); + if (ret == CRYPTO_SUCCESS) { + data->cd_length = data->cd_offset - saved_offset; + } else { + data->cd_length = saved_length; + } + + data->cd_offset = saved_offset; + if (ret != CRYPTO_SUCCESS) { + return (ret); + } + } + + + if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) { + data->cd_length = 0; + } + + (void) aes_free_context(ctx); + + return (CRYPTO_SUCCESS); +} + +/* ARGSUSED */ +static int +aes_encrypt_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext, + crypto_spi_ctx_template_t template, crypto_req_handle_t req) +{ + aes_ctx_t aes_ctx; /* on the stack */ + off_t saved_offset; + size_t saved_length; + size_t length_needed; + int ret; + + AES_ARG_INPLACE(plaintext, ciphertext); + + /* + * CTR, CCM, GCM, and GMAC modes do not require that plaintext + * be a multiple of AES block size. + */ + switch (mechanism->cm_type) { + case AES_CTR_MECH_INFO_TYPE: + case AES_CCM_MECH_INFO_TYPE: + case AES_GCM_MECH_INFO_TYPE: + case AES_GMAC_MECH_INFO_TYPE: + break; + default: + if ((plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0) + return (CRYPTO_DATA_LEN_RANGE); + } + + if ((ret = aes_check_mech_param(mechanism, NULL, 0)) != CRYPTO_SUCCESS) + return (ret); + + bzero(&aes_ctx, sizeof (aes_ctx_t)); + + ret = aes_common_init_ctx(&aes_ctx, template, mechanism, key, + crypto_kmflag(req), B_TRUE); + if (ret != CRYPTO_SUCCESS) + return (ret); + + switch (mechanism->cm_type) { + case AES_CCM_MECH_INFO_TYPE: + length_needed = plaintext->cd_length + aes_ctx.ac_mac_len; + break; + case AES_GMAC_MECH_INFO_TYPE: + if (plaintext->cd_length != 0) + return (CRYPTO_ARGUMENTS_BAD); + /* FALLTHRU */ + case AES_GCM_MECH_INFO_TYPE: + length_needed = plaintext->cd_length + aes_ctx.ac_tag_len; + break; + default: + length_needed = plaintext->cd_length; + } + + /* return size of buffer needed to store output */ + if (ciphertext->cd_length < length_needed) { + ciphertext->cd_length = length_needed; + ret = CRYPTO_BUFFER_TOO_SMALL; + goto out; + } + + saved_offset = ciphertext->cd_offset; + saved_length = ciphertext->cd_length; + + /* + * Do an update on the specified input data. + */ + switch (plaintext->cd_format) { + case CRYPTO_DATA_RAW: + ret = crypto_update_iov(&aes_ctx, plaintext, ciphertext, + aes_encrypt_contiguous_blocks, aes_copy_block64); + break; + case CRYPTO_DATA_UIO: + ret = crypto_update_uio(&aes_ctx, plaintext, ciphertext, + aes_encrypt_contiguous_blocks, aes_copy_block64); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret == CRYPTO_SUCCESS) { + if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) { + ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx, + ciphertext, AES_BLOCK_LEN, aes_encrypt_block, + aes_xor_block); + if (ret != CRYPTO_SUCCESS) + goto out; + ASSERT(aes_ctx.ac_remainder_len == 0); + } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE || + mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) { + ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx, + ciphertext, AES_BLOCK_LEN, aes_encrypt_block, + aes_copy_block, aes_xor_block); + if (ret != CRYPTO_SUCCESS) + goto out; + ASSERT(aes_ctx.ac_remainder_len == 0); + } else if (mechanism->cm_type == AES_CTR_MECH_INFO_TYPE) { + if (aes_ctx.ac_remainder_len > 0) { + ret = ctr_mode_final((ctr_ctx_t *)&aes_ctx, + ciphertext, aes_encrypt_block); + if (ret != CRYPTO_SUCCESS) + goto out; + } + } else { + ASSERT(aes_ctx.ac_remainder_len == 0); + } + + if (plaintext != ciphertext) { + ciphertext->cd_length = + ciphertext->cd_offset - saved_offset; + } + } else { + ciphertext->cd_length = saved_length; + } + ciphertext->cd_offset = saved_offset; + +out: + if (aes_ctx.ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) { + bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len); + kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len); + } + + return (ret); +} + +/* ARGSUSED */ +static int +aes_decrypt_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext, + crypto_spi_ctx_template_t template, crypto_req_handle_t req) +{ + aes_ctx_t aes_ctx; /* on the stack */ + off_t saved_offset; + size_t saved_length; + size_t length_needed; + int ret; + + AES_ARG_INPLACE(ciphertext, plaintext); + + /* + * CCM, GCM, CTR, and GMAC modes do not require that ciphertext + * be a multiple of AES block size. + */ + switch (mechanism->cm_type) { + case AES_CTR_MECH_INFO_TYPE: + case AES_CCM_MECH_INFO_TYPE: + case AES_GCM_MECH_INFO_TYPE: + case AES_GMAC_MECH_INFO_TYPE: + break; + default: + if ((ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0) + return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); + } + + if ((ret = aes_check_mech_param(mechanism, NULL, 0)) != CRYPTO_SUCCESS) + return (ret); + + bzero(&aes_ctx, sizeof (aes_ctx_t)); + + ret = aes_common_init_ctx(&aes_ctx, template, mechanism, key, + crypto_kmflag(req), B_FALSE); + if (ret != CRYPTO_SUCCESS) + return (ret); + + switch (mechanism->cm_type) { + case AES_CCM_MECH_INFO_TYPE: + length_needed = aes_ctx.ac_data_len; + break; + case AES_GCM_MECH_INFO_TYPE: + length_needed = ciphertext->cd_length - aes_ctx.ac_tag_len; + break; + case AES_GMAC_MECH_INFO_TYPE: + if (plaintext->cd_length != 0) + return (CRYPTO_ARGUMENTS_BAD); + length_needed = 0; + break; + default: + length_needed = ciphertext->cd_length; + } + + /* return size of buffer needed to store output */ + if (plaintext->cd_length < length_needed) { + plaintext->cd_length = length_needed; + ret = CRYPTO_BUFFER_TOO_SMALL; + goto out; + } + + saved_offset = plaintext->cd_offset; + saved_length = plaintext->cd_length; + + if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE || + mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) + gcm_set_kmflag((gcm_ctx_t *)&aes_ctx, crypto_kmflag(req)); + + /* + * Do an update on the specified input data. + */ + switch (ciphertext->cd_format) { + case CRYPTO_DATA_RAW: + ret = crypto_update_iov(&aes_ctx, ciphertext, plaintext, + aes_decrypt_contiguous_blocks, aes_copy_block64); + break; + case CRYPTO_DATA_UIO: + ret = crypto_update_uio(&aes_ctx, ciphertext, plaintext, + aes_decrypt_contiguous_blocks, aes_copy_block64); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret == CRYPTO_SUCCESS) { + if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) { + ASSERT(aes_ctx.ac_processed_data_len + == aes_ctx.ac_data_len); + ASSERT(aes_ctx.ac_processed_mac_len + == aes_ctx.ac_mac_len); + ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx, + plaintext, AES_BLOCK_LEN, aes_encrypt_block, + aes_copy_block, aes_xor_block); + ASSERT(aes_ctx.ac_remainder_len == 0); + if ((ret == CRYPTO_SUCCESS) && + (ciphertext != plaintext)) { + plaintext->cd_length = + plaintext->cd_offset - saved_offset; + } else { + plaintext->cd_length = saved_length; + } + } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE || + mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) { + ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx, + plaintext, AES_BLOCK_LEN, aes_encrypt_block, + aes_xor_block); + ASSERT(aes_ctx.ac_remainder_len == 0); + if ((ret == CRYPTO_SUCCESS) && + (ciphertext != plaintext)) { + plaintext->cd_length = + plaintext->cd_offset - saved_offset; + } else { + plaintext->cd_length = saved_length; + } + } else if (mechanism->cm_type != AES_CTR_MECH_INFO_TYPE) { + ASSERT(aes_ctx.ac_remainder_len == 0); + if (ciphertext != plaintext) + plaintext->cd_length = + plaintext->cd_offset - saved_offset; + } else { + if (aes_ctx.ac_remainder_len > 0) { + ret = ctr_mode_final((ctr_ctx_t *)&aes_ctx, + plaintext, aes_encrypt_block); + if (ret == CRYPTO_DATA_LEN_RANGE) + ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; + if (ret != CRYPTO_SUCCESS) + goto out; + } + if (ciphertext != plaintext) + plaintext->cd_length = + plaintext->cd_offset - saved_offset; + } + } else { + plaintext->cd_length = saved_length; + } + plaintext->cd_offset = saved_offset; + +out: + if (aes_ctx.ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) { + bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len); + kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len); + } + + if (aes_ctx.ac_flags & CCM_MODE) { + if (aes_ctx.ac_pt_buf != NULL) { + vmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len); + } + } else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) { + if (((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf != NULL) { + vmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf, + ((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len); + } + } + + return (ret); +} + +/* + * KCF software provider context template entry points. + */ +/* ARGSUSED */ +static int +aes_create_ctx_template(crypto_provider_handle_t provider, + crypto_mechanism_t *mechanism, crypto_key_t *key, + crypto_spi_ctx_template_t *tmpl, size_t *tmpl_size, crypto_req_handle_t req) +{ + void *keysched; + size_t size; + int rv; + + if (mechanism->cm_type != AES_ECB_MECH_INFO_TYPE && + mechanism->cm_type != AES_CBC_MECH_INFO_TYPE && + mechanism->cm_type != AES_CTR_MECH_INFO_TYPE && + mechanism->cm_type != AES_CCM_MECH_INFO_TYPE && + mechanism->cm_type != AES_GCM_MECH_INFO_TYPE && + mechanism->cm_type != AES_GMAC_MECH_INFO_TYPE) + return (CRYPTO_MECHANISM_INVALID); + + if ((keysched = aes_alloc_keysched(&size, + crypto_kmflag(req))) == NULL) { + return (CRYPTO_HOST_MEMORY); + } + + /* + * Initialize key schedule. Key length information is stored + * in the key. + */ + if ((rv = init_keysched(key, keysched)) != CRYPTO_SUCCESS) { + bzero(keysched, size); + kmem_free(keysched, size); + return (rv); + } + + *tmpl = keysched; + *tmpl_size = size; + + return (CRYPTO_SUCCESS); +} + + +static int +aes_free_context(crypto_ctx_t *ctx) +{ + aes_ctx_t *aes_ctx = ctx->cc_provider_private; + + if (aes_ctx != NULL) { + if (aes_ctx->ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) { + ASSERT(aes_ctx->ac_keysched_len != 0); + bzero(aes_ctx->ac_keysched, aes_ctx->ac_keysched_len); + kmem_free(aes_ctx->ac_keysched, + aes_ctx->ac_keysched_len); + } + crypto_free_mode_ctx(aes_ctx); + ctx->cc_provider_private = NULL; + } + + return (CRYPTO_SUCCESS); +} + + +static int +aes_common_init_ctx(aes_ctx_t *aes_ctx, crypto_spi_ctx_template_t *template, + crypto_mechanism_t *mechanism, crypto_key_t *key, int kmflag, + boolean_t is_encrypt_init) +{ + int rv = CRYPTO_SUCCESS; + void *keysched; + size_t size; + + if (template == NULL) { + if ((keysched = aes_alloc_keysched(&size, kmflag)) == NULL) + return (CRYPTO_HOST_MEMORY); + /* + * Initialize key schedule. + * Key length is stored in the key. + */ + if ((rv = init_keysched(key, keysched)) != CRYPTO_SUCCESS) { + kmem_free(keysched, size); + return (rv); + } + + aes_ctx->ac_flags |= PROVIDER_OWNS_KEY_SCHEDULE; + aes_ctx->ac_keysched_len = size; + } else { + keysched = template; + } + aes_ctx->ac_keysched = keysched; + + switch (mechanism->cm_type) { + case AES_CBC_MECH_INFO_TYPE: + rv = cbc_init_ctx((cbc_ctx_t *)aes_ctx, mechanism->cm_param, + mechanism->cm_param_len, AES_BLOCK_LEN, aes_copy_block64); + break; + case AES_CTR_MECH_INFO_TYPE: { + CK_AES_CTR_PARAMS *pp; + + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) { + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param; + rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits, + pp->cb, aes_copy_block); + break; + } + case AES_CCM_MECH_INFO_TYPE: + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) { + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param, + kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block, + aes_xor_block); + break; + case AES_GCM_MECH_INFO_TYPE: + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (CK_AES_GCM_PARAMS)) { + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param, + AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + break; + case AES_GMAC_MECH_INFO_TYPE: + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) { + return (CRYPTO_MECHANISM_PARAM_INVALID); + } + rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param, + AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, + aes_xor_block); + break; + case AES_ECB_MECH_INFO_TYPE: + aes_ctx->ac_flags |= ECB_MODE; + } + + if (rv != CRYPTO_SUCCESS) { + if (aes_ctx->ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) { + bzero(keysched, size); + kmem_free(keysched, size); + } + } + + return (rv); +} + +static int +process_gmac_mech(crypto_mechanism_t *mech, crypto_data_t *data, + CK_AES_GCM_PARAMS *gcm_params) +{ + /* LINTED: pointer alignment */ + CK_AES_GMAC_PARAMS *params = (CK_AES_GMAC_PARAMS *)mech->cm_param; + + if (mech->cm_type != AES_GMAC_MECH_INFO_TYPE) + return (CRYPTO_MECHANISM_INVALID); + + if (mech->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) + return (CRYPTO_MECHANISM_PARAM_INVALID); + + if (params->pIv == NULL) + return (CRYPTO_MECHANISM_PARAM_INVALID); + + gcm_params->pIv = params->pIv; + gcm_params->ulIvLen = AES_GMAC_IV_LEN; + gcm_params->ulTagBits = AES_GMAC_TAG_BITS; + + if (data == NULL) + return (CRYPTO_SUCCESS); + + if (data->cd_format != CRYPTO_DATA_RAW) + return (CRYPTO_ARGUMENTS_BAD); + + gcm_params->pAAD = (uchar_t *)data->cd_raw.iov_base; + gcm_params->ulAADLen = data->cd_length; + return (CRYPTO_SUCCESS); +} + +static int +aes_mac_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, + crypto_spi_ctx_template_t template, crypto_req_handle_t req) +{ + CK_AES_GCM_PARAMS gcm_params; + crypto_mechanism_t gcm_mech; + int rv; + + if ((rv = process_gmac_mech(mechanism, data, &gcm_params)) + != CRYPTO_SUCCESS) + return (rv); + + gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE; + gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); + gcm_mech.cm_param = (char *)&gcm_params; + + return (aes_encrypt_atomic(provider, session_id, &gcm_mech, + key, &null_crypto_data, mac, template, req)); +} + +static int +aes_mac_verify_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, + crypto_spi_ctx_template_t template, crypto_req_handle_t req) +{ + CK_AES_GCM_PARAMS gcm_params; + crypto_mechanism_t gcm_mech; + int rv; + + if ((rv = process_gmac_mech(mechanism, data, &gcm_params)) + != CRYPTO_SUCCESS) + return (rv); + + gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE; + gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); + gcm_mech.cm_param = (char *)&gcm_params; + + return (aes_decrypt_atomic(provider, session_id, &gcm_mech, + key, mac, &null_crypto_data, template, req)); +} diff --git a/module/icp/io/sha1_mod.c b/module/icp/io/sha1_mod.c new file mode 100644 index 000000000000..a278dac7fc82 --- /dev/null +++ b/module/icp/io/sha1_mod.c @@ -0,0 +1,1239 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2010 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include + +#include +#include + +/* + * The sha1 module is created with two modlinkages: + * - a modlmisc that allows consumers to directly call the entry points + * SHA1Init, SHA1Update, and SHA1Final. + * - a modlcrypto that allows the module to register with the Kernel + * Cryptographic Framework (KCF) as a software provider for the SHA1 + * mechanisms. + */ + +static struct modlcrypto modlcrypto = { + &mod_cryptoops, + "SHA1 Kernel SW Provider 1.1" +}; + +static struct modlinkage modlinkage = { + MODREV_1, { &modlcrypto, NULL } +}; + + +/* + * Macros to access the SHA1 or SHA1-HMAC contexts from a context passed + * by KCF to one of the entry points. + */ + +#define PROV_SHA1_CTX(ctx) ((sha1_ctx_t *)(ctx)->cc_provider_private) +#define PROV_SHA1_HMAC_CTX(ctx) ((sha1_hmac_ctx_t *)(ctx)->cc_provider_private) + +/* to extract the digest length passed as mechanism parameter */ +#define PROV_SHA1_GET_DIGEST_LEN(m, len) { \ + if (IS_P2ALIGNED((m)->cm_param, sizeof (ulong_t))) \ + (len) = (uint32_t)*((ulong_t *)(void *)mechanism->cm_param); \ + else { \ + ulong_t tmp_ulong; \ + bcopy((m)->cm_param, &tmp_ulong, sizeof (ulong_t)); \ + (len) = (uint32_t)tmp_ulong; \ + } \ +} + +#define PROV_SHA1_DIGEST_KEY(ctx, key, len, digest) { \ + SHA1Init(ctx); \ + SHA1Update(ctx, key, len); \ + SHA1Final(digest, ctx); \ +} + +/* + * Mechanism info structure passed to KCF during registration. + */ +static crypto_mech_info_t sha1_mech_info_tab[] = { + /* SHA1 */ + {SUN_CKM_SHA1, SHA1_MECH_INFO_TYPE, + CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC, + 0, 0, CRYPTO_KEYSIZE_UNIT_IN_BITS}, + /* SHA1-HMAC */ + {SUN_CKM_SHA1_HMAC, SHA1_HMAC_MECH_INFO_TYPE, + CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC, + SHA1_HMAC_MIN_KEY_LEN, SHA1_HMAC_MAX_KEY_LEN, + CRYPTO_KEYSIZE_UNIT_IN_BYTES}, + /* SHA1-HMAC GENERAL */ + {SUN_CKM_SHA1_HMAC_GENERAL, SHA1_HMAC_GEN_MECH_INFO_TYPE, + CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC, + SHA1_HMAC_MIN_KEY_LEN, SHA1_HMAC_MAX_KEY_LEN, + CRYPTO_KEYSIZE_UNIT_IN_BYTES} +}; + +static void sha1_provider_status(crypto_provider_handle_t, uint_t *); + +static crypto_control_ops_t sha1_control_ops = { + sha1_provider_status +}; + +static int sha1_digest_init(crypto_ctx_t *, crypto_mechanism_t *, + crypto_req_handle_t); +static int sha1_digest(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, + crypto_req_handle_t); +static int sha1_digest_update(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); +static int sha1_digest_final(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); +static int sha1_digest_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_data_t *, crypto_data_t *, + crypto_req_handle_t); + +static crypto_digest_ops_t sha1_digest_ops = { + sha1_digest_init, + sha1_digest, + sha1_digest_update, + NULL, + sha1_digest_final, + sha1_digest_atomic +}; + +static int sha1_mac_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); +static int sha1_mac_update(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); +static int sha1_mac_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); +static int sha1_mac_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); +static int sha1_mac_verify_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); + +static crypto_mac_ops_t sha1_mac_ops = { + sha1_mac_init, + NULL, + sha1_mac_update, + sha1_mac_final, + sha1_mac_atomic, + sha1_mac_verify_atomic +}; + +static int sha1_create_ctx_template(crypto_provider_handle_t, + crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *, + size_t *, crypto_req_handle_t); +static int sha1_free_context(crypto_ctx_t *); + +static crypto_ctx_ops_t sha1_ctx_ops = { + sha1_create_ctx_template, + sha1_free_context +}; + +static crypto_ops_t sha1_crypto_ops = {{{{{ + &sha1_control_ops, + &sha1_digest_ops, + NULL, + &sha1_mac_ops, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + &sha1_ctx_ops, +}}}}}; + +static crypto_provider_info_t sha1_prov_info = {{{{ + CRYPTO_SPI_VERSION_1, + "SHA1 Software Provider", + CRYPTO_SW_PROVIDER, + NULL, + &sha1_crypto_ops, + sizeof (sha1_mech_info_tab)/sizeof (crypto_mech_info_t), + sha1_mech_info_tab +}}}}; + +static crypto_kcf_provider_handle_t sha1_prov_handle = 0; + +int +sha1_mod_init(void) +{ + int ret; + + if ((ret = mod_install(&modlinkage)) != 0) + return (ret); + + /* + * Register with KCF. If the registration fails, log an + * error but do not uninstall the module, since the functionality + * provided by misc/sha1 should still be available. + */ + if ((ret = crypto_register_provider(&sha1_prov_info, + &sha1_prov_handle)) != CRYPTO_SUCCESS) + cmn_err(CE_WARN, "sha1 _init: " + "crypto_register_provider() failed (0x%x)", ret); + + return (0); +} + +int +sha1_mod_fini(void) +{ + int ret; + + if (sha1_prov_handle != 0) { + if ((ret = crypto_unregister_provider(sha1_prov_handle)) != + CRYPTO_SUCCESS) { + cmn_err(CE_WARN, + "sha1 _fini: crypto_unregister_provider() " + "failed (0x%x)", ret); + return (EBUSY); + } + sha1_prov_handle = 0; + } + + return (mod_remove(&modlinkage)); +} + +/* + * KCF software provider control entry points. + */ +/* ARGSUSED */ +static void +sha1_provider_status(crypto_provider_handle_t provider, uint_t *status) +{ + *status = CRYPTO_PROVIDER_READY; +} + +/* + * KCF software provider digest entry points. + */ + +static int +sha1_digest_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, + crypto_req_handle_t req) +{ + if (mechanism->cm_type != SHA1_MECH_INFO_TYPE) + return (CRYPTO_MECHANISM_INVALID); + + /* + * Allocate and initialize SHA1 context. + */ + ctx->cc_provider_private = kmem_alloc(sizeof (sha1_ctx_t), + crypto_kmflag(req)); + if (ctx->cc_provider_private == NULL) + return (CRYPTO_HOST_MEMORY); + + PROV_SHA1_CTX(ctx)->sc_mech_type = SHA1_MECH_INFO_TYPE; + SHA1Init(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx); + + return (CRYPTO_SUCCESS); +} + +/* + * Helper SHA1 digest update function for uio data. + */ +static int +sha1_digest_update_uio(SHA1_CTX *sha1_ctx, crypto_data_t *data) +{ + off_t offset = data->cd_offset; + size_t length = data->cd_length; + uint_t vec_idx; + size_t cur_len; + + /* we support only kernel buffer */ + if (data->cd_uio->uio_segflg != UIO_SYSSPACE) + return (CRYPTO_ARGUMENTS_BAD); + + /* + * Jump to the first iovec containing data to be + * digested. + */ + for (vec_idx = 0; vec_idx < data->cd_uio->uio_iovcnt && + offset >= data->cd_uio->uio_iov[vec_idx].iov_len; + offset -= data->cd_uio->uio_iov[vec_idx++].iov_len) + ; + if (vec_idx == data->cd_uio->uio_iovcnt) { + /* + * The caller specified an offset that is larger than the + * total size of the buffers it provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + + /* + * Now do the digesting on the iovecs. + */ + while (vec_idx < data->cd_uio->uio_iovcnt && length > 0) { + cur_len = MIN(data->cd_uio->uio_iov[vec_idx].iov_len - + offset, length); + + SHA1Update(sha1_ctx, + (uint8_t *)data->cd_uio->uio_iov[vec_idx].iov_base + offset, + cur_len); + + length -= cur_len; + vec_idx++; + offset = 0; + } + + if (vec_idx == data->cd_uio->uio_iovcnt && length > 0) { + /* + * The end of the specified iovec's was reached but + * the length requested could not be processed, i.e. + * The caller requested to digest more data than it provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + + return (CRYPTO_SUCCESS); +} + +/* + * Helper SHA1 digest final function for uio data. + * digest_len is the length of the desired digest. If digest_len + * is smaller than the default SHA1 digest length, the caller + * must pass a scratch buffer, digest_scratch, which must + * be at least SHA1_DIGEST_LENGTH bytes. + */ +static int +sha1_digest_final_uio(SHA1_CTX *sha1_ctx, crypto_data_t *digest, + ulong_t digest_len, uchar_t *digest_scratch) +{ + off_t offset = digest->cd_offset; + uint_t vec_idx; + + /* we support only kernel buffer */ + if (digest->cd_uio->uio_segflg != UIO_SYSSPACE) + return (CRYPTO_ARGUMENTS_BAD); + + /* + * Jump to the first iovec containing ptr to the digest to + * be returned. + */ + for (vec_idx = 0; offset >= digest->cd_uio->uio_iov[vec_idx].iov_len && + vec_idx < digest->cd_uio->uio_iovcnt; + offset -= digest->cd_uio->uio_iov[vec_idx++].iov_len) + ; + if (vec_idx == digest->cd_uio->uio_iovcnt) { + /* + * The caller specified an offset that is + * larger than the total size of the buffers + * it provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + + if (offset + digest_len <= + digest->cd_uio->uio_iov[vec_idx].iov_len) { + /* + * The computed SHA1 digest will fit in the current + * iovec. + */ + if (digest_len != SHA1_DIGEST_LENGTH) { + /* + * The caller requested a short digest. Digest + * into a scratch buffer and return to + * the user only what was requested. + */ + SHA1Final(digest_scratch, sha1_ctx); + bcopy(digest_scratch, (uchar_t *)digest-> + cd_uio->uio_iov[vec_idx].iov_base + offset, + digest_len); + } else { + SHA1Final((uchar_t *)digest-> + cd_uio->uio_iov[vec_idx].iov_base + offset, + sha1_ctx); + } + } else { + /* + * The computed digest will be crossing one or more iovec's. + * This is bad performance-wise but we need to support it. + * Allocate a small scratch buffer on the stack and + * copy it piece meal to the specified digest iovec's. + */ + uchar_t digest_tmp[SHA1_DIGEST_LENGTH]; + off_t scratch_offset = 0; + size_t length = digest_len; + size_t cur_len; + + SHA1Final(digest_tmp, sha1_ctx); + + while (vec_idx < digest->cd_uio->uio_iovcnt && length > 0) { + cur_len = MIN(digest->cd_uio->uio_iov[vec_idx].iov_len - + offset, length); + bcopy(digest_tmp + scratch_offset, + digest->cd_uio->uio_iov[vec_idx].iov_base + offset, + cur_len); + + length -= cur_len; + vec_idx++; + scratch_offset += cur_len; + offset = 0; + } + + if (vec_idx == digest->cd_uio->uio_iovcnt && length > 0) { + /* + * The end of the specified iovec's was reached but + * the length requested could not be processed, i.e. + * The caller requested to digest more data than it + * provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + } + + return (CRYPTO_SUCCESS); +} + +/* ARGSUSED */ +static int +sha1_digest(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *digest, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + + ASSERT(ctx->cc_provider_private != NULL); + + /* + * We need to just return the length needed to store the output. + * We should not destroy the context for the following cases. + */ + if ((digest->cd_length == 0) || + (digest->cd_length < SHA1_DIGEST_LENGTH)) { + digest->cd_length = SHA1_DIGEST_LENGTH; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + /* + * Do the SHA1 update on the specified input data. + */ + switch (data->cd_format) { + case CRYPTO_DATA_RAW: + SHA1Update(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx, + (uint8_t *)data->cd_raw.iov_base + data->cd_offset, + data->cd_length); + break; + case CRYPTO_DATA_UIO: + ret = sha1_digest_update_uio(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx, + data); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret != CRYPTO_SUCCESS) { + /* the update failed, free context and bail */ + kmem_free(ctx->cc_provider_private, sizeof (sha1_ctx_t)); + ctx->cc_provider_private = NULL; + digest->cd_length = 0; + return (ret); + } + + /* + * Do a SHA1 final, must be done separately since the digest + * type can be different than the input data type. + */ + switch (digest->cd_format) { + case CRYPTO_DATA_RAW: + SHA1Final((unsigned char *)digest->cd_raw.iov_base + + digest->cd_offset, &PROV_SHA1_CTX(ctx)->sc_sha1_ctx); + break; + case CRYPTO_DATA_UIO: + ret = sha1_digest_final_uio(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx, + digest, SHA1_DIGEST_LENGTH, NULL); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + /* all done, free context and return */ + + if (ret == CRYPTO_SUCCESS) { + digest->cd_length = SHA1_DIGEST_LENGTH; + } else { + digest->cd_length = 0; + } + + kmem_free(ctx->cc_provider_private, sizeof (sha1_ctx_t)); + ctx->cc_provider_private = NULL; + return (ret); +} + +/* ARGSUSED */ +static int +sha1_digest_update(crypto_ctx_t *ctx, crypto_data_t *data, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + + ASSERT(ctx->cc_provider_private != NULL); + + /* + * Do the SHA1 update on the specified input data. + */ + switch (data->cd_format) { + case CRYPTO_DATA_RAW: + SHA1Update(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx, + (uint8_t *)data->cd_raw.iov_base + data->cd_offset, + data->cd_length); + break; + case CRYPTO_DATA_UIO: + ret = sha1_digest_update_uio(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx, + data); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + return (ret); +} + +/* ARGSUSED */ +static int +sha1_digest_final(crypto_ctx_t *ctx, crypto_data_t *digest, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + + ASSERT(ctx->cc_provider_private != NULL); + + /* + * We need to just return the length needed to store the output. + * We should not destroy the context for the following cases. + */ + if ((digest->cd_length == 0) || + (digest->cd_length < SHA1_DIGEST_LENGTH)) { + digest->cd_length = SHA1_DIGEST_LENGTH; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + /* + * Do a SHA1 final. + */ + switch (digest->cd_format) { + case CRYPTO_DATA_RAW: + SHA1Final((unsigned char *)digest->cd_raw.iov_base + + digest->cd_offset, &PROV_SHA1_CTX(ctx)->sc_sha1_ctx); + break; + case CRYPTO_DATA_UIO: + ret = sha1_digest_final_uio(&PROV_SHA1_CTX(ctx)->sc_sha1_ctx, + digest, SHA1_DIGEST_LENGTH, NULL); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + /* all done, free context and return */ + + if (ret == CRYPTO_SUCCESS) { + digest->cd_length = SHA1_DIGEST_LENGTH; + } else { + digest->cd_length = 0; + } + + kmem_free(ctx->cc_provider_private, sizeof (sha1_ctx_t)); + ctx->cc_provider_private = NULL; + + return (ret); +} + +/* ARGSUSED */ +static int +sha1_digest_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_data_t *data, crypto_data_t *digest, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + SHA1_CTX sha1_ctx; + + if (mechanism->cm_type != SHA1_MECH_INFO_TYPE) + return (CRYPTO_MECHANISM_INVALID); + + /* + * Do the SHA1 init. + */ + SHA1Init(&sha1_ctx); + + /* + * Do the SHA1 update on the specified input data. + */ + switch (data->cd_format) { + case CRYPTO_DATA_RAW: + SHA1Update(&sha1_ctx, + (uint8_t *)data->cd_raw.iov_base + data->cd_offset, + data->cd_length); + break; + case CRYPTO_DATA_UIO: + ret = sha1_digest_update_uio(&sha1_ctx, data); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret != CRYPTO_SUCCESS) { + /* the update failed, bail */ + digest->cd_length = 0; + return (ret); + } + + /* + * Do a SHA1 final, must be done separately since the digest + * type can be different than the input data type. + */ + switch (digest->cd_format) { + case CRYPTO_DATA_RAW: + SHA1Final((unsigned char *)digest->cd_raw.iov_base + + digest->cd_offset, &sha1_ctx); + break; + case CRYPTO_DATA_UIO: + ret = sha1_digest_final_uio(&sha1_ctx, digest, + SHA1_DIGEST_LENGTH, NULL); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret == CRYPTO_SUCCESS) { + digest->cd_length = SHA1_DIGEST_LENGTH; + } else { + digest->cd_length = 0; + } + + return (ret); +} + +/* + * KCF software provider mac entry points. + * + * SHA1 HMAC is: SHA1(key XOR opad, SHA1(key XOR ipad, text)) + * + * Init: + * The initialization routine initializes what we denote + * as the inner and outer contexts by doing + * - for inner context: SHA1(key XOR ipad) + * - for outer context: SHA1(key XOR opad) + * + * Update: + * Each subsequent SHA1 HMAC update will result in an + * update of the inner context with the specified data. + * + * Final: + * The SHA1 HMAC final will do a SHA1 final operation on the + * inner context, and the resulting digest will be used + * as the data for an update on the outer context. Last + * but not least, a SHA1 final on the outer context will + * be performed to obtain the SHA1 HMAC digest to return + * to the user. + */ + +/* + * Initialize a SHA1-HMAC context. + */ +static void +sha1_mac_init_ctx(sha1_hmac_ctx_t *ctx, void *keyval, uint_t length_in_bytes) +{ + uint32_t ipad[SHA1_HMAC_INTS_PER_BLOCK]; + uint32_t opad[SHA1_HMAC_INTS_PER_BLOCK]; + uint_t i; + + bzero(ipad, SHA1_HMAC_BLOCK_SIZE); + bzero(opad, SHA1_HMAC_BLOCK_SIZE); + + bcopy(keyval, ipad, length_in_bytes); + bcopy(keyval, opad, length_in_bytes); + + /* XOR key with ipad (0x36) and opad (0x5c) */ + for (i = 0; i < SHA1_HMAC_INTS_PER_BLOCK; i++) { + ipad[i] ^= 0x36363636; + opad[i] ^= 0x5c5c5c5c; + } + + /* perform SHA1 on ipad */ + SHA1Init(&ctx->hc_icontext); + SHA1Update(&ctx->hc_icontext, (uint8_t *)ipad, SHA1_HMAC_BLOCK_SIZE); + + /* perform SHA1 on opad */ + SHA1Init(&ctx->hc_ocontext); + SHA1Update(&ctx->hc_ocontext, (uint8_t *)opad, SHA1_HMAC_BLOCK_SIZE); +} + +/* + */ +static int +sha1_mac_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length); + + if (mechanism->cm_type != SHA1_HMAC_MECH_INFO_TYPE && + mechanism->cm_type != SHA1_HMAC_GEN_MECH_INFO_TYPE) + return (CRYPTO_MECHANISM_INVALID); + + /* Add support for key by attributes (RFE 4706552) */ + if (key->ck_format != CRYPTO_KEY_RAW) + return (CRYPTO_ARGUMENTS_BAD); + + ctx->cc_provider_private = kmem_alloc(sizeof (sha1_hmac_ctx_t), + crypto_kmflag(req)); + if (ctx->cc_provider_private == NULL) + return (CRYPTO_HOST_MEMORY); + + if (ctx_template != NULL) { + /* reuse context template */ + bcopy(ctx_template, PROV_SHA1_HMAC_CTX(ctx), + sizeof (sha1_hmac_ctx_t)); + } else { + /* no context template, compute context */ + if (keylen_in_bytes > SHA1_HMAC_BLOCK_SIZE) { + uchar_t digested_key[SHA1_DIGEST_LENGTH]; + sha1_hmac_ctx_t *hmac_ctx = ctx->cc_provider_private; + + /* + * Hash the passed-in key to get a smaller key. + * The inner context is used since it hasn't been + * initialized yet. + */ + PROV_SHA1_DIGEST_KEY(&hmac_ctx->hc_icontext, + key->ck_data, keylen_in_bytes, digested_key); + sha1_mac_init_ctx(PROV_SHA1_HMAC_CTX(ctx), + digested_key, SHA1_DIGEST_LENGTH); + } else { + sha1_mac_init_ctx(PROV_SHA1_HMAC_CTX(ctx), + key->ck_data, keylen_in_bytes); + } + } + + /* + * Get the mechanism parameters, if applicable. + */ + PROV_SHA1_HMAC_CTX(ctx)->hc_mech_type = mechanism->cm_type; + if (mechanism->cm_type == SHA1_HMAC_GEN_MECH_INFO_TYPE) { + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (ulong_t)) + ret = CRYPTO_MECHANISM_PARAM_INVALID; + PROV_SHA1_GET_DIGEST_LEN(mechanism, + PROV_SHA1_HMAC_CTX(ctx)->hc_digest_len); + if (PROV_SHA1_HMAC_CTX(ctx)->hc_digest_len > + SHA1_DIGEST_LENGTH) + ret = CRYPTO_MECHANISM_PARAM_INVALID; + } + + if (ret != CRYPTO_SUCCESS) { + bzero(ctx->cc_provider_private, sizeof (sha1_hmac_ctx_t)); + kmem_free(ctx->cc_provider_private, sizeof (sha1_hmac_ctx_t)); + ctx->cc_provider_private = NULL; + } + + return (ret); +} + +/* ARGSUSED */ +static int +sha1_mac_update(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + + ASSERT(ctx->cc_provider_private != NULL); + + /* + * Do a SHA1 update of the inner context using the specified + * data. + */ + switch (data->cd_format) { + case CRYPTO_DATA_RAW: + SHA1Update(&PROV_SHA1_HMAC_CTX(ctx)->hc_icontext, + (uint8_t *)data->cd_raw.iov_base + data->cd_offset, + data->cd_length); + break; + case CRYPTO_DATA_UIO: + ret = sha1_digest_update_uio( + &PROV_SHA1_HMAC_CTX(ctx)->hc_icontext, data); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + return (ret); +} + +/* ARGSUSED */ +static int +sha1_mac_final(crypto_ctx_t *ctx, crypto_data_t *mac, crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uchar_t digest[SHA1_DIGEST_LENGTH]; + uint32_t digest_len = SHA1_DIGEST_LENGTH; + + ASSERT(ctx->cc_provider_private != NULL); + + if (PROV_SHA1_HMAC_CTX(ctx)->hc_mech_type == + SHA1_HMAC_GEN_MECH_INFO_TYPE) + digest_len = PROV_SHA1_HMAC_CTX(ctx)->hc_digest_len; + + /* + * We need to just return the length needed to store the output. + * We should not destroy the context for the following cases. + */ + if ((mac->cd_length == 0) || (mac->cd_length < digest_len)) { + mac->cd_length = digest_len; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + /* + * Do a SHA1 final on the inner context. + */ + SHA1Final(digest, &PROV_SHA1_HMAC_CTX(ctx)->hc_icontext); + + /* + * Do a SHA1 update on the outer context, feeding the inner + * digest as data. + */ + SHA1Update(&PROV_SHA1_HMAC_CTX(ctx)->hc_ocontext, digest, + SHA1_DIGEST_LENGTH); + + /* + * Do a SHA1 final on the outer context, storing the computing + * digest in the users buffer. + */ + switch (mac->cd_format) { + case CRYPTO_DATA_RAW: + if (digest_len != SHA1_DIGEST_LENGTH) { + /* + * The caller requested a short digest. Digest + * into a scratch buffer and return to + * the user only what was requested. + */ + SHA1Final(digest, + &PROV_SHA1_HMAC_CTX(ctx)->hc_ocontext); + bcopy(digest, (unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, digest_len); + } else { + SHA1Final((unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, + &PROV_SHA1_HMAC_CTX(ctx)->hc_ocontext); + } + break; + case CRYPTO_DATA_UIO: + ret = sha1_digest_final_uio( + &PROV_SHA1_HMAC_CTX(ctx)->hc_ocontext, mac, + digest_len, digest); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret == CRYPTO_SUCCESS) { + mac->cd_length = digest_len; + } else { + mac->cd_length = 0; + } + + bzero(ctx->cc_provider_private, sizeof (sha1_hmac_ctx_t)); + kmem_free(ctx->cc_provider_private, sizeof (sha1_hmac_ctx_t)); + ctx->cc_provider_private = NULL; + + return (ret); +} + +#define SHA1_MAC_UPDATE(data, ctx, ret) { \ + switch (data->cd_format) { \ + case CRYPTO_DATA_RAW: \ + SHA1Update(&(ctx).hc_icontext, \ + (uint8_t *)data->cd_raw.iov_base + \ + data->cd_offset, data->cd_length); \ + break; \ + case CRYPTO_DATA_UIO: \ + ret = sha1_digest_update_uio(&(ctx).hc_icontext, data); \ + break; \ + default: \ + ret = CRYPTO_ARGUMENTS_BAD; \ + } \ +} + +/* ARGSUSED */ +static int +sha1_mac_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, + crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uchar_t digest[SHA1_DIGEST_LENGTH]; + sha1_hmac_ctx_t sha1_hmac_ctx; + uint32_t digest_len = SHA1_DIGEST_LENGTH; + uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length); + + if (mechanism->cm_type != SHA1_HMAC_MECH_INFO_TYPE && + mechanism->cm_type != SHA1_HMAC_GEN_MECH_INFO_TYPE) + return (CRYPTO_MECHANISM_INVALID); + + /* Add support for key by attributes (RFE 4706552) */ + if (key->ck_format != CRYPTO_KEY_RAW) + return (CRYPTO_ARGUMENTS_BAD); + + if (ctx_template != NULL) { + /* reuse context template */ + bcopy(ctx_template, &sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t)); + } else { + /* no context template, initialize context */ + if (keylen_in_bytes > SHA1_HMAC_BLOCK_SIZE) { + /* + * Hash the passed-in key to get a smaller key. + * The inner context is used since it hasn't been + * initialized yet. + */ + PROV_SHA1_DIGEST_KEY(&sha1_hmac_ctx.hc_icontext, + key->ck_data, keylen_in_bytes, digest); + sha1_mac_init_ctx(&sha1_hmac_ctx, digest, + SHA1_DIGEST_LENGTH); + } else { + sha1_mac_init_ctx(&sha1_hmac_ctx, key->ck_data, + keylen_in_bytes); + } + } + + /* get the mechanism parameters, if applicable */ + if (mechanism->cm_type == SHA1_HMAC_GEN_MECH_INFO_TYPE) { + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (ulong_t)) { + ret = CRYPTO_MECHANISM_PARAM_INVALID; + goto bail; + } + PROV_SHA1_GET_DIGEST_LEN(mechanism, digest_len); + if (digest_len > SHA1_DIGEST_LENGTH) { + ret = CRYPTO_MECHANISM_PARAM_INVALID; + goto bail; + } + } + + /* do a SHA1 update of the inner context using the specified data */ + SHA1_MAC_UPDATE(data, sha1_hmac_ctx, ret); + if (ret != CRYPTO_SUCCESS) + /* the update failed, free context and bail */ + goto bail; + + /* + * Do a SHA1 final on the inner context. + */ + SHA1Final(digest, &sha1_hmac_ctx.hc_icontext); + + /* + * Do an SHA1 update on the outer context, feeding the inner + * digest as data. + */ + SHA1Update(&sha1_hmac_ctx.hc_ocontext, digest, SHA1_DIGEST_LENGTH); + + /* + * Do a SHA1 final on the outer context, storing the computed + * digest in the users buffer. + */ + switch (mac->cd_format) { + case CRYPTO_DATA_RAW: + if (digest_len != SHA1_DIGEST_LENGTH) { + /* + * The caller requested a short digest. Digest + * into a scratch buffer and return to + * the user only what was requested. + */ + SHA1Final(digest, &sha1_hmac_ctx.hc_ocontext); + bcopy(digest, (unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, digest_len); + } else { + SHA1Final((unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, &sha1_hmac_ctx.hc_ocontext); + } + break; + case CRYPTO_DATA_UIO: + ret = sha1_digest_final_uio(&sha1_hmac_ctx.hc_ocontext, mac, + digest_len, digest); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret == CRYPTO_SUCCESS) { + mac->cd_length = digest_len; + } else { + mac->cd_length = 0; + } + /* Extra paranoia: zeroize the context on the stack */ + bzero(&sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t)); + + return (ret); +bail: + bzero(&sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t)); + mac->cd_length = 0; + return (ret); +} + +/* ARGSUSED */ +static int +sha1_mac_verify_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, + crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uchar_t digest[SHA1_DIGEST_LENGTH]; + sha1_hmac_ctx_t sha1_hmac_ctx; + uint32_t digest_len = SHA1_DIGEST_LENGTH; + uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length); + + if (mechanism->cm_type != SHA1_HMAC_MECH_INFO_TYPE && + mechanism->cm_type != SHA1_HMAC_GEN_MECH_INFO_TYPE) + return (CRYPTO_MECHANISM_INVALID); + + /* Add support for key by attributes (RFE 4706552) */ + if (key->ck_format != CRYPTO_KEY_RAW) + return (CRYPTO_ARGUMENTS_BAD); + + if (ctx_template != NULL) { + /* reuse context template */ + bcopy(ctx_template, &sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t)); + } else { + /* no context template, initialize context */ + if (keylen_in_bytes > SHA1_HMAC_BLOCK_SIZE) { + /* + * Hash the passed-in key to get a smaller key. + * The inner context is used since it hasn't been + * initialized yet. + */ + PROV_SHA1_DIGEST_KEY(&sha1_hmac_ctx.hc_icontext, + key->ck_data, keylen_in_bytes, digest); + sha1_mac_init_ctx(&sha1_hmac_ctx, digest, + SHA1_DIGEST_LENGTH); + } else { + sha1_mac_init_ctx(&sha1_hmac_ctx, key->ck_data, + keylen_in_bytes); + } + } + + /* get the mechanism parameters, if applicable */ + if (mechanism->cm_type == SHA1_HMAC_GEN_MECH_INFO_TYPE) { + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (ulong_t)) { + ret = CRYPTO_MECHANISM_PARAM_INVALID; + goto bail; + } + PROV_SHA1_GET_DIGEST_LEN(mechanism, digest_len); + if (digest_len > SHA1_DIGEST_LENGTH) { + ret = CRYPTO_MECHANISM_PARAM_INVALID; + goto bail; + } + } + + if (mac->cd_length != digest_len) { + ret = CRYPTO_INVALID_MAC; + goto bail; + } + + /* do a SHA1 update of the inner context using the specified data */ + SHA1_MAC_UPDATE(data, sha1_hmac_ctx, ret); + if (ret != CRYPTO_SUCCESS) + /* the update failed, free context and bail */ + goto bail; + + /* do a SHA1 final on the inner context */ + SHA1Final(digest, &sha1_hmac_ctx.hc_icontext); + + /* + * Do an SHA1 update on the outer context, feeding the inner + * digest as data. + */ + SHA1Update(&sha1_hmac_ctx.hc_ocontext, digest, SHA1_DIGEST_LENGTH); + + /* + * Do a SHA1 final on the outer context, storing the computed + * digest in the users buffer. + */ + SHA1Final(digest, &sha1_hmac_ctx.hc_ocontext); + + /* + * Compare the computed digest against the expected digest passed + * as argument. + */ + + switch (mac->cd_format) { + + case CRYPTO_DATA_RAW: + if (bcmp(digest, (unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, digest_len) != 0) + ret = CRYPTO_INVALID_MAC; + break; + + case CRYPTO_DATA_UIO: { + off_t offset = mac->cd_offset; + uint_t vec_idx; + off_t scratch_offset = 0; + size_t length = digest_len; + size_t cur_len; + + /* we support only kernel buffer */ + if (mac->cd_uio->uio_segflg != UIO_SYSSPACE) + return (CRYPTO_ARGUMENTS_BAD); + + /* jump to the first iovec containing the expected digest */ + for (vec_idx = 0; + offset >= mac->cd_uio->uio_iov[vec_idx].iov_len && + vec_idx < mac->cd_uio->uio_iovcnt; + offset -= mac->cd_uio->uio_iov[vec_idx++].iov_len) + ; + if (vec_idx == mac->cd_uio->uio_iovcnt) { + /* + * The caller specified an offset that is + * larger than the total size of the buffers + * it provided. + */ + ret = CRYPTO_DATA_LEN_RANGE; + break; + } + + /* do the comparison of computed digest vs specified one */ + while (vec_idx < mac->cd_uio->uio_iovcnt && length > 0) { + cur_len = MIN(mac->cd_uio->uio_iov[vec_idx].iov_len - + offset, length); + + if (bcmp(digest + scratch_offset, + mac->cd_uio->uio_iov[vec_idx].iov_base + offset, + cur_len) != 0) { + ret = CRYPTO_INVALID_MAC; + break; + } + + length -= cur_len; + vec_idx++; + scratch_offset += cur_len; + offset = 0; + } + break; + } + + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + bzero(&sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t)); + return (ret); +bail: + bzero(&sha1_hmac_ctx, sizeof (sha1_hmac_ctx_t)); + mac->cd_length = 0; + return (ret); +} + +/* + * KCF software provider context management entry points. + */ + +/* ARGSUSED */ +static int +sha1_create_ctx_template(crypto_provider_handle_t provider, + crypto_mechanism_t *mechanism, crypto_key_t *key, + crypto_spi_ctx_template_t *ctx_template, size_t *ctx_template_size, + crypto_req_handle_t req) +{ + sha1_hmac_ctx_t *sha1_hmac_ctx_tmpl; + uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length); + + if ((mechanism->cm_type != SHA1_HMAC_MECH_INFO_TYPE) && + (mechanism->cm_type != SHA1_HMAC_GEN_MECH_INFO_TYPE)) { + return (CRYPTO_MECHANISM_INVALID); + } + + /* Add support for key by attributes (RFE 4706552) */ + if (key->ck_format != CRYPTO_KEY_RAW) + return (CRYPTO_ARGUMENTS_BAD); + + /* + * Allocate and initialize SHA1 context. + */ + sha1_hmac_ctx_tmpl = kmem_alloc(sizeof (sha1_hmac_ctx_t), + crypto_kmflag(req)); + if (sha1_hmac_ctx_tmpl == NULL) + return (CRYPTO_HOST_MEMORY); + + if (keylen_in_bytes > SHA1_HMAC_BLOCK_SIZE) { + uchar_t digested_key[SHA1_DIGEST_LENGTH]; + + /* + * Hash the passed-in key to get a smaller key. + * The inner context is used since it hasn't been + * initialized yet. + */ + PROV_SHA1_DIGEST_KEY(&sha1_hmac_ctx_tmpl->hc_icontext, + key->ck_data, keylen_in_bytes, digested_key); + sha1_mac_init_ctx(sha1_hmac_ctx_tmpl, digested_key, + SHA1_DIGEST_LENGTH); + } else { + sha1_mac_init_ctx(sha1_hmac_ctx_tmpl, key->ck_data, + keylen_in_bytes); + } + + sha1_hmac_ctx_tmpl->hc_mech_type = mechanism->cm_type; + *ctx_template = (crypto_spi_ctx_template_t)sha1_hmac_ctx_tmpl; + *ctx_template_size = sizeof (sha1_hmac_ctx_t); + + + return (CRYPTO_SUCCESS); +} + +static int +sha1_free_context(crypto_ctx_t *ctx) +{ + uint_t ctx_len; + sha1_mech_type_t mech_type; + + if (ctx->cc_provider_private == NULL) + return (CRYPTO_SUCCESS); + + /* + * We have to free either SHA1 or SHA1-HMAC contexts, which + * have different lengths. + */ + + mech_type = PROV_SHA1_CTX(ctx)->sc_mech_type; + if (mech_type == SHA1_MECH_INFO_TYPE) + ctx_len = sizeof (sha1_ctx_t); + else { + ASSERT(mech_type == SHA1_HMAC_MECH_INFO_TYPE || + mech_type == SHA1_HMAC_GEN_MECH_INFO_TYPE); + ctx_len = sizeof (sha1_hmac_ctx_t); + } + + bzero(ctx->cc_provider_private, ctx_len); + kmem_free(ctx->cc_provider_private, ctx_len); + ctx->cc_provider_private = NULL; + + return (CRYPTO_SUCCESS); +} diff --git a/module/icp/io/sha2_mod.c b/module/icp/io/sha2_mod.c new file mode 100644 index 000000000000..fc5f9604442e --- /dev/null +++ b/module/icp/io/sha2_mod.c @@ -0,0 +1,1307 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright 2010 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include +#include +#include +#include +#define _SHA2_IMPL +#include +#include + +/* + * The sha2 module is created with two modlinkages: + * - a modlmisc that allows consumers to directly call the entry points + * SHA2Init, SHA2Update, and SHA2Final. + * - a modlcrypto that allows the module to register with the Kernel + * Cryptographic Framework (KCF) as a software provider for the SHA2 + * mechanisms. + */ + +static struct modlcrypto modlcrypto = { + &mod_cryptoops, + "SHA2 Kernel SW Provider" +}; + +static struct modlinkage modlinkage = { + MODREV_1, {&modlcrypto, NULL} +}; + +/* + * Macros to access the SHA2 or SHA2-HMAC contexts from a context passed + * by KCF to one of the entry points. + */ + +#define PROV_SHA2_CTX(ctx) ((sha2_ctx_t *)(ctx)->cc_provider_private) +#define PROV_SHA2_HMAC_CTX(ctx) ((sha2_hmac_ctx_t *)(ctx)->cc_provider_private) + +/* to extract the digest length passed as mechanism parameter */ +#define PROV_SHA2_GET_DIGEST_LEN(m, len) { \ + if (IS_P2ALIGNED((m)->cm_param, sizeof (ulong_t))) \ + (len) = (uint32_t)*((ulong_t *)(m)->cm_param); \ + else { \ + ulong_t tmp_ulong; \ + bcopy((m)->cm_param, &tmp_ulong, sizeof (ulong_t)); \ + (len) = (uint32_t)tmp_ulong; \ + } \ +} + +#define PROV_SHA2_DIGEST_KEY(mech, ctx, key, len, digest) { \ + SHA2Init(mech, ctx); \ + SHA2Update(ctx, key, len); \ + SHA2Final(digest, ctx); \ +} + +/* + * Mechanism info structure passed to KCF during registration. + */ +static crypto_mech_info_t sha2_mech_info_tab[] = { + /* SHA256 */ + {SUN_CKM_SHA256, SHA256_MECH_INFO_TYPE, + CRYPTO_FG_DIGEST | CRYPTO_FG_DIGEST_ATOMIC, + 0, 0, CRYPTO_KEYSIZE_UNIT_IN_BITS}, + /* SHA256-HMAC */ + {SUN_CKM_SHA256_HMAC, SHA256_HMAC_MECH_INFO_TYPE, + CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC, + SHA2_HMAC_MIN_KEY_LEN, SHA2_HMAC_MAX_KEY_LEN, + CRYPTO_KEYSIZE_UNIT_IN_BYTES}, + /* SHA256-HMAC GENERAL */ + {SUN_CKM_SHA256_HMAC_GENERAL, SHA256_HMAC_GEN_MECH_INFO_TYPE, + CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC, + SHA2_HMAC_MIN_KEY_LEN, SHA2_HMAC_MAX_KEY_LEN, + CRYPTO_KEYSIZE_UNIT_IN_BYTES} +}; + +static void sha2_provider_status(crypto_provider_handle_t, uint_t *); + +static crypto_control_ops_t sha2_control_ops = { + sha2_provider_status +}; + +static int sha2_digest_init(crypto_ctx_t *, crypto_mechanism_t *, + crypto_req_handle_t); +static int sha2_digest(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, + crypto_req_handle_t); +static int sha2_digest_update(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); +static int sha2_digest_final(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); +static int sha2_digest_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_data_t *, crypto_data_t *, + crypto_req_handle_t); + +static crypto_digest_ops_t sha2_digest_ops = { + sha2_digest_init, + sha2_digest, + sha2_digest_update, + NULL, + sha2_digest_final, + sha2_digest_atomic +}; + +static int sha2_mac_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); +static int sha2_mac_update(crypto_ctx_t *, crypto_data_t *, + crypto_req_handle_t); +static int sha2_mac_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); +static int sha2_mac_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); +static int sha2_mac_verify_atomic(crypto_provider_handle_t, crypto_session_id_t, + crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, + crypto_spi_ctx_template_t, crypto_req_handle_t); + +static crypto_mac_ops_t sha2_mac_ops = { + sha2_mac_init, + NULL, + sha2_mac_update, + sha2_mac_final, + sha2_mac_atomic, + sha2_mac_verify_atomic +}; + +static int sha2_create_ctx_template(crypto_provider_handle_t, + crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *, + size_t *, crypto_req_handle_t); +static int sha2_free_context(crypto_ctx_t *); + +static crypto_ctx_ops_t sha2_ctx_ops = { + sha2_create_ctx_template, + sha2_free_context +}; + +static crypto_ops_t sha2_crypto_ops = {{{{{ + &sha2_control_ops, + &sha2_digest_ops, + NULL, + &sha2_mac_ops, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + &sha2_ctx_ops +}}}}}; + +static crypto_provider_info_t sha2_prov_info = {{{{ + CRYPTO_SPI_VERSION_1, + "SHA2 Software Provider", + CRYPTO_SW_PROVIDER, + NULL, + &sha2_crypto_ops, + sizeof (sha2_mech_info_tab)/sizeof (crypto_mech_info_t), + sha2_mech_info_tab +}}}}; + +static crypto_kcf_provider_handle_t sha2_prov_handle = 0; + +int +sha2_mod_init(void) +{ + int ret; + + if ((ret = mod_install(&modlinkage)) != 0) + return (ret); + + /* + * Register with KCF. If the registration fails, log an + * error but do not uninstall the module, since the functionality + * provided by misc/sha2 should still be available. + */ + if ((ret = crypto_register_provider(&sha2_prov_info, + &sha2_prov_handle)) != CRYPTO_SUCCESS) + cmn_err(CE_WARN, "sha2 _init: " + "crypto_register_provider() failed (0x%x)", ret); + + return (0); +} + +int +sha2_mod_fini(void) +{ + int ret; + + if (sha2_prov_handle != 0) { + if ((ret = crypto_unregister_provider(sha2_prov_handle)) != + CRYPTO_SUCCESS) { + cmn_err(CE_WARN, + "sha2 _fini: crypto_unregister_provider() " + "failed (0x%x)", ret); + return (EBUSY); + } + sha2_prov_handle = 0; + } + + return (mod_remove(&modlinkage)); +} + +/* + * KCF software provider control entry points. + */ +/* ARGSUSED */ +static void +sha2_provider_status(crypto_provider_handle_t provider, uint_t *status) +{ + *status = CRYPTO_PROVIDER_READY; +} + +/* + * KCF software provider digest entry points. + */ + +static int +sha2_digest_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, + crypto_req_handle_t req) +{ + + /* + * Allocate and initialize SHA2 context. + */ + ctx->cc_provider_private = kmem_alloc(sizeof (sha2_ctx_t), + crypto_kmflag(req)); + if (ctx->cc_provider_private == NULL) + return (CRYPTO_HOST_MEMORY); + + PROV_SHA2_CTX(ctx)->sc_mech_type = mechanism->cm_type; + SHA2Init(mechanism->cm_type, &PROV_SHA2_CTX(ctx)->sc_sha2_ctx); + + return (CRYPTO_SUCCESS); +} + +/* + * Helper SHA2 digest update function for uio data. + */ +static int +sha2_digest_update_uio(SHA2_CTX *sha2_ctx, crypto_data_t *data) +{ + off_t offset = data->cd_offset; + size_t length = data->cd_length; + uint_t vec_idx; + size_t cur_len; + + /* we support only kernel buffer */ + if (data->cd_uio->uio_segflg != UIO_SYSSPACE) + return (CRYPTO_ARGUMENTS_BAD); + + /* + * Jump to the first iovec containing data to be + * digested. + */ + for (vec_idx = 0; vec_idx < data->cd_uio->uio_iovcnt && + offset >= data->cd_uio->uio_iov[vec_idx].iov_len; + offset -= data->cd_uio->uio_iov[vec_idx++].iov_len) + ; + if (vec_idx == data->cd_uio->uio_iovcnt) { + /* + * The caller specified an offset that is larger than the + * total size of the buffers it provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + + /* + * Now do the digesting on the iovecs. + */ + while (vec_idx < data->cd_uio->uio_iovcnt && length > 0) { + cur_len = MIN(data->cd_uio->uio_iov[vec_idx].iov_len - + offset, length); + + SHA2Update(sha2_ctx, (uint8_t *)data->cd_uio-> + uio_iov[vec_idx].iov_base + offset, cur_len); + length -= cur_len; + vec_idx++; + offset = 0; + } + + if (vec_idx == data->cd_uio->uio_iovcnt && length > 0) { + /* + * The end of the specified iovec's was reached but + * the length requested could not be processed, i.e. + * The caller requested to digest more data than it provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + + return (CRYPTO_SUCCESS); +} + +/* + * Helper SHA2 digest final function for uio data. + * digest_len is the length of the desired digest. If digest_len + * is smaller than the default SHA2 digest length, the caller + * must pass a scratch buffer, digest_scratch, which must + * be at least the algorithm's digest length bytes. + */ +static int +sha2_digest_final_uio(SHA2_CTX *sha2_ctx, crypto_data_t *digest, + ulong_t digest_len, uchar_t *digest_scratch) +{ + off_t offset = digest->cd_offset; + uint_t vec_idx; + + /* we support only kernel buffer */ + if (digest->cd_uio->uio_segflg != UIO_SYSSPACE) + return (CRYPTO_ARGUMENTS_BAD); + + /* + * Jump to the first iovec containing ptr to the digest to + * be returned. + */ + for (vec_idx = 0; offset >= digest->cd_uio->uio_iov[vec_idx].iov_len && + vec_idx < digest->cd_uio->uio_iovcnt; + offset -= digest->cd_uio->uio_iov[vec_idx++].iov_len) + ; + if (vec_idx == digest->cd_uio->uio_iovcnt) { + /* + * The caller specified an offset that is + * larger than the total size of the buffers + * it provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + + if (offset + digest_len <= + digest->cd_uio->uio_iov[vec_idx].iov_len) { + /* + * The computed SHA2 digest will fit in the current + * iovec. + */ + if (((sha2_ctx->algotype <= SHA256_HMAC_GEN_MECH_INFO_TYPE) && + (digest_len != SHA256_DIGEST_LENGTH))) { + /* + * The caller requested a short digest. Digest + * into a scratch buffer and return to + * the user only what was requested. + */ + SHA2Final(digest_scratch, sha2_ctx); + + bcopy(digest_scratch, (uchar_t *)digest-> + cd_uio->uio_iov[vec_idx].iov_base + offset, + digest_len); + } else { + SHA2Final((uchar_t *)digest-> + cd_uio->uio_iov[vec_idx].iov_base + offset, + sha2_ctx); + + } + } else { + /* + * The computed digest will be crossing one or more iovec's. + * This is bad performance-wise but we need to support it. + * Allocate a small scratch buffer on the stack and + * copy it piece meal to the specified digest iovec's. + */ + uchar_t digest_tmp[SHA256_DIGEST_LENGTH]; + off_t scratch_offset = 0; + size_t length = digest_len; + size_t cur_len; + + SHA2Final(digest_tmp, sha2_ctx); + + while (vec_idx < digest->cd_uio->uio_iovcnt && length > 0) { + cur_len = + MIN(digest->cd_uio->uio_iov[vec_idx].iov_len - + offset, length); + bcopy(digest_tmp + scratch_offset, + digest->cd_uio->uio_iov[vec_idx].iov_base + offset, + cur_len); + + length -= cur_len; + vec_idx++; + scratch_offset += cur_len; + offset = 0; + } + + if (vec_idx == digest->cd_uio->uio_iovcnt && length > 0) { + /* + * The end of the specified iovec's was reached but + * the length requested could not be processed, i.e. + * The caller requested to digest more data than it + * provided. + */ + return (CRYPTO_DATA_LEN_RANGE); + } + } + + return (CRYPTO_SUCCESS); +} + +/* ARGSUSED */ +static int +sha2_digest(crypto_ctx_t *ctx, crypto_data_t *data, crypto_data_t *digest, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uint_t sha_digest_len; + + ASSERT(ctx->cc_provider_private != NULL); + + switch (PROV_SHA2_CTX(ctx)->sc_mech_type) { + case SHA256_MECH_INFO_TYPE: + sha_digest_len = SHA256_DIGEST_LENGTH; + break; + default: + return (CRYPTO_MECHANISM_INVALID); + } + + /* + * We need to just return the length needed to store the output. + * We should not destroy the context for the following cases. + */ + if ((digest->cd_length == 0) || + (digest->cd_length < sha_digest_len)) { + digest->cd_length = sha_digest_len; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + /* + * Do the SHA2 update on the specified input data. + */ + switch (data->cd_format) { + case CRYPTO_DATA_RAW: + SHA2Update(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, + (uint8_t *)data->cd_raw.iov_base + data->cd_offset, + data->cd_length); + break; + case CRYPTO_DATA_UIO: + ret = sha2_digest_update_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, + data); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret != CRYPTO_SUCCESS) { + /* the update failed, free context and bail */ + kmem_free(ctx->cc_provider_private, sizeof (sha2_ctx_t)); + ctx->cc_provider_private = NULL; + digest->cd_length = 0; + return (ret); + } + + /* + * Do a SHA2 final, must be done separately since the digest + * type can be different than the input data type. + */ + switch (digest->cd_format) { + case CRYPTO_DATA_RAW: + SHA2Final((unsigned char *)digest->cd_raw.iov_base + + digest->cd_offset, &PROV_SHA2_CTX(ctx)->sc_sha2_ctx); + break; + case CRYPTO_DATA_UIO: + ret = sha2_digest_final_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, + digest, sha_digest_len, NULL); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + /* all done, free context and return */ + + if (ret == CRYPTO_SUCCESS) + digest->cd_length = sha_digest_len; + else + digest->cd_length = 0; + + kmem_free(ctx->cc_provider_private, sizeof (sha2_ctx_t)); + ctx->cc_provider_private = NULL; + return (ret); +} + +/* ARGSUSED */ +static int +sha2_digest_update(crypto_ctx_t *ctx, crypto_data_t *data, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + + ASSERT(ctx->cc_provider_private != NULL); + + /* + * Do the SHA2 update on the specified input data. + */ + switch (data->cd_format) { + case CRYPTO_DATA_RAW: + SHA2Update(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, + (uint8_t *)data->cd_raw.iov_base + data->cd_offset, + data->cd_length); + break; + case CRYPTO_DATA_UIO: + ret = sha2_digest_update_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, + data); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + return (ret); +} + +/* ARGSUSED */ +static int +sha2_digest_final(crypto_ctx_t *ctx, crypto_data_t *digest, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uint_t sha_digest_len; + + ASSERT(ctx->cc_provider_private != NULL); + + switch (PROV_SHA2_CTX(ctx)->sc_mech_type) { + case SHA256_MECH_INFO_TYPE: + sha_digest_len = SHA256_DIGEST_LENGTH; + break; + default: + return (CRYPTO_MECHANISM_INVALID); + } + + /* + * We need to just return the length needed to store the output. + * We should not destroy the context for the following cases. + */ + if ((digest->cd_length == 0) || + (digest->cd_length < sha_digest_len)) { + digest->cd_length = sha_digest_len; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + /* + * Do a SHA2 final. + */ + switch (digest->cd_format) { + case CRYPTO_DATA_RAW: + SHA2Final((unsigned char *)digest->cd_raw.iov_base + + digest->cd_offset, &PROV_SHA2_CTX(ctx)->sc_sha2_ctx); + break; + case CRYPTO_DATA_UIO: + ret = sha2_digest_final_uio(&PROV_SHA2_CTX(ctx)->sc_sha2_ctx, + digest, sha_digest_len, NULL); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + /* all done, free context and return */ + + if (ret == CRYPTO_SUCCESS) + digest->cd_length = sha_digest_len; + else + digest->cd_length = 0; + + kmem_free(ctx->cc_provider_private, sizeof (sha2_ctx_t)); + ctx->cc_provider_private = NULL; + + return (ret); +} + +/* ARGSUSED */ +static int +sha2_digest_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_data_t *data, crypto_data_t *digest, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + SHA2_CTX sha2_ctx; + uint32_t sha_digest_len; + + /* + * Do the SHA inits. + */ + + SHA2Init(mechanism->cm_type, &sha2_ctx); + + switch (data->cd_format) { + case CRYPTO_DATA_RAW: + SHA2Update(&sha2_ctx, (uint8_t *)data-> + cd_raw.iov_base + data->cd_offset, data->cd_length); + break; + case CRYPTO_DATA_UIO: + ret = sha2_digest_update_uio(&sha2_ctx, data); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + /* + * Do the SHA updates on the specified input data. + */ + + if (ret != CRYPTO_SUCCESS) { + /* the update failed, bail */ + digest->cd_length = 0; + return (ret); + } + + if (mechanism->cm_type <= SHA256_HMAC_GEN_MECH_INFO_TYPE) + sha_digest_len = SHA256_DIGEST_LENGTH; + + /* + * Do a SHA2 final, must be done separately since the digest + * type can be different than the input data type. + */ + switch (digest->cd_format) { + case CRYPTO_DATA_RAW: + SHA2Final((unsigned char *)digest->cd_raw.iov_base + + digest->cd_offset, &sha2_ctx); + break; + case CRYPTO_DATA_UIO: + ret = sha2_digest_final_uio(&sha2_ctx, digest, + sha_digest_len, NULL); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret == CRYPTO_SUCCESS) + digest->cd_length = sha_digest_len; + else + digest->cd_length = 0; + + return (ret); +} + +/* + * KCF software provider mac entry points. + * + * SHA2 HMAC is: SHA2(key XOR opad, SHA2(key XOR ipad, text)) + * + * Init: + * The initialization routine initializes what we denote + * as the inner and outer contexts by doing + * - for inner context: SHA2(key XOR ipad) + * - for outer context: SHA2(key XOR opad) + * + * Update: + * Each subsequent SHA2 HMAC update will result in an + * update of the inner context with the specified data. + * + * Final: + * The SHA2 HMAC final will do a SHA2 final operation on the + * inner context, and the resulting digest will be used + * as the data for an update on the outer context. Last + * but not least, a SHA2 final on the outer context will + * be performed to obtain the SHA2 HMAC digest to return + * to the user. + */ + +/* + * Initialize a SHA2-HMAC context. + */ +static void +sha2_mac_init_ctx(sha2_hmac_ctx_t *ctx, void *keyval, uint_t length_in_bytes) +{ + uint64_t ipad[SHA256_HMAC_BLOCK_SIZE / sizeof (uint64_t)]; + uint64_t opad[SHA256_HMAC_BLOCK_SIZE / sizeof (uint64_t)]; + int i, block_size, blocks_per_int64 = 0; + + /* Determine the block size */ + if (ctx->hc_mech_type <= SHA256_HMAC_GEN_MECH_INFO_TYPE) { + block_size = SHA256_HMAC_BLOCK_SIZE; + blocks_per_int64 = SHA256_HMAC_BLOCK_SIZE / sizeof (uint64_t); + } + + (void) bzero(ipad, block_size); + (void) bzero(opad, block_size); + (void) bcopy(keyval, ipad, length_in_bytes); + (void) bcopy(keyval, opad, length_in_bytes); + + /* XOR key with ipad (0x36) and opad (0x5c) */ + for (i = 0; i < blocks_per_int64; i ++) { + ipad[i] ^= 0x3636363636363636; + opad[i] ^= 0x5c5c5c5c5c5c5c5c; + } + + /* perform SHA2 on ipad */ + SHA2Init(ctx->hc_mech_type, &ctx->hc_icontext); + SHA2Update(&ctx->hc_icontext, (uint8_t *)ipad, block_size); + + /* perform SHA2 on opad */ + SHA2Init(ctx->hc_mech_type, &ctx->hc_ocontext); + SHA2Update(&ctx->hc_ocontext, (uint8_t *)opad, block_size); + +} + +/* + */ +static int +sha2_mac_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_spi_ctx_template_t ctx_template, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length); + uint_t sha_digest_len, sha_hmac_block_size; + + /* + * Set the digest length and block size to values approriate to the + * mechanism + */ + switch (mechanism->cm_type) { + case SHA256_HMAC_MECH_INFO_TYPE: + case SHA256_HMAC_GEN_MECH_INFO_TYPE: + sha_digest_len = SHA256_DIGEST_LENGTH; + sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE; + break; + default: + return (CRYPTO_MECHANISM_INVALID); + } + + if (key->ck_format != CRYPTO_KEY_RAW) + return (CRYPTO_ARGUMENTS_BAD); + + ctx->cc_provider_private = kmem_alloc(sizeof (sha2_hmac_ctx_t), + crypto_kmflag(req)); + if (ctx->cc_provider_private == NULL) + return (CRYPTO_HOST_MEMORY); + + PROV_SHA2_HMAC_CTX(ctx)->hc_mech_type = mechanism->cm_type; + if (ctx_template != NULL) { + /* reuse context template */ + bcopy(ctx_template, PROV_SHA2_HMAC_CTX(ctx), + sizeof (sha2_hmac_ctx_t)); + } else { + /* no context template, compute context */ + if (keylen_in_bytes > sha_hmac_block_size) { + uchar_t digested_key[SHA256_DIGEST_LENGTH]; + sha2_hmac_ctx_t *hmac_ctx = ctx->cc_provider_private; + + /* + * Hash the passed-in key to get a smaller key. + * The inner context is used since it hasn't been + * initialized yet. + */ + PROV_SHA2_DIGEST_KEY(mechanism->cm_type / 3, + &hmac_ctx->hc_icontext, + key->ck_data, keylen_in_bytes, digested_key); + sha2_mac_init_ctx(PROV_SHA2_HMAC_CTX(ctx), + digested_key, sha_digest_len); + } else { + sha2_mac_init_ctx(PROV_SHA2_HMAC_CTX(ctx), + key->ck_data, keylen_in_bytes); + } + } + + /* + * Get the mechanism parameters, if applicable. + */ + if (mechanism->cm_type % 3 == 2) { + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (ulong_t)) + ret = CRYPTO_MECHANISM_PARAM_INVALID; + PROV_SHA2_GET_DIGEST_LEN(mechanism, + PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len); + if (PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len > sha_digest_len) + ret = CRYPTO_MECHANISM_PARAM_INVALID; + } + + if (ret != CRYPTO_SUCCESS) { + bzero(ctx->cc_provider_private, sizeof (sha2_hmac_ctx_t)); + kmem_free(ctx->cc_provider_private, sizeof (sha2_hmac_ctx_t)); + ctx->cc_provider_private = NULL; + } + + return (ret); +} + +/* ARGSUSED */ +static int +sha2_mac_update(crypto_ctx_t *ctx, crypto_data_t *data, + crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + + ASSERT(ctx->cc_provider_private != NULL); + + /* + * Do a SHA2 update of the inner context using the specified + * data. + */ + switch (data->cd_format) { + case CRYPTO_DATA_RAW: + SHA2Update(&PROV_SHA2_HMAC_CTX(ctx)->hc_icontext, + (uint8_t *)data->cd_raw.iov_base + data->cd_offset, + data->cd_length); + break; + case CRYPTO_DATA_UIO: + ret = sha2_digest_update_uio( + &PROV_SHA2_HMAC_CTX(ctx)->hc_icontext, data); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + return (ret); +} + +/* ARGSUSED */ +static int +sha2_mac_final(crypto_ctx_t *ctx, crypto_data_t *mac, crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uchar_t digest[SHA256_DIGEST_LENGTH]; + uint32_t digest_len = 0, sha_digest_len = 0; + + ASSERT(ctx->cc_provider_private != NULL); + + /* Set the digest lengths to values approriate to the mechanism */ + switch (PROV_SHA2_HMAC_CTX(ctx)->hc_mech_type) { + case SHA256_HMAC_MECH_INFO_TYPE: + sha_digest_len = digest_len = SHA256_DIGEST_LENGTH; + break; + case SHA256_HMAC_GEN_MECH_INFO_TYPE: + sha_digest_len = SHA256_DIGEST_LENGTH; + digest_len = PROV_SHA2_HMAC_CTX(ctx)->hc_digest_len; + break; + default: + break; + } + + /* + * We need to just return the length needed to store the output. + * We should not destroy the context for the following cases. + */ + if ((mac->cd_length == 0) || (mac->cd_length < digest_len)) { + mac->cd_length = digest_len; + return (CRYPTO_BUFFER_TOO_SMALL); + } + + /* + * Do a SHA2 final on the inner context. + */ + SHA2Final(digest, &PROV_SHA2_HMAC_CTX(ctx)->hc_icontext); + + /* + * Do a SHA2 update on the outer context, feeding the inner + * digest as data. + */ + SHA2Update(&PROV_SHA2_HMAC_CTX(ctx)->hc_ocontext, digest, + sha_digest_len); + + /* + * Do a SHA2 final on the outer context, storing the computing + * digest in the users buffer. + */ + switch (mac->cd_format) { + case CRYPTO_DATA_RAW: + if (digest_len != sha_digest_len) { + /* + * The caller requested a short digest. Digest + * into a scratch buffer and return to + * the user only what was requested. + */ + SHA2Final(digest, + &PROV_SHA2_HMAC_CTX(ctx)->hc_ocontext); + bcopy(digest, (unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, digest_len); + } else { + SHA2Final((unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, + &PROV_SHA2_HMAC_CTX(ctx)->hc_ocontext); + } + break; + case CRYPTO_DATA_UIO: + ret = sha2_digest_final_uio( + &PROV_SHA2_HMAC_CTX(ctx)->hc_ocontext, mac, + digest_len, digest); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret == CRYPTO_SUCCESS) + mac->cd_length = digest_len; + else + mac->cd_length = 0; + + bzero(ctx->cc_provider_private, sizeof (sha2_hmac_ctx_t)); + kmem_free(ctx->cc_provider_private, sizeof (sha2_hmac_ctx_t)); + ctx->cc_provider_private = NULL; + + return (ret); +} + +#define SHA2_MAC_UPDATE(data, ctx, ret) { \ + switch (data->cd_format) { \ + case CRYPTO_DATA_RAW: \ + SHA2Update(&(ctx).hc_icontext, \ + (uint8_t *)data->cd_raw.iov_base + \ + data->cd_offset, data->cd_length); \ + break; \ + case CRYPTO_DATA_UIO: \ + ret = sha2_digest_update_uio(&(ctx).hc_icontext, data); \ + break; \ + default: \ + ret = CRYPTO_ARGUMENTS_BAD; \ + } \ +} + +/* ARGSUSED */ +static int +sha2_mac_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, + crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uchar_t digest[SHA256_DIGEST_LENGTH]; + sha2_hmac_ctx_t sha2_hmac_ctx; + uint32_t sha_digest_len, digest_len, sha_hmac_block_size; + uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length); + + /* + * Set the digest length and block size to values appropriate to the + * mechanism + */ + switch (mechanism->cm_type) { + case SHA256_HMAC_MECH_INFO_TYPE: + case SHA256_HMAC_GEN_MECH_INFO_TYPE: + sha_digest_len = digest_len = SHA256_DIGEST_LENGTH; + sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE; + break; + default: + return (CRYPTO_MECHANISM_INVALID); + } + + /* Add support for key by attributes (RFE 4706552) */ + if (key->ck_format != CRYPTO_KEY_RAW) + return (CRYPTO_ARGUMENTS_BAD); + + if (ctx_template != NULL) { + /* reuse context template */ + bcopy(ctx_template, &sha2_hmac_ctx, sizeof (sha2_hmac_ctx_t)); + } else { + sha2_hmac_ctx.hc_mech_type = mechanism->cm_type; + /* no context template, initialize context */ + if (keylen_in_bytes > sha_hmac_block_size) { + /* + * Hash the passed-in key to get a smaller key. + * The inner context is used since it hasn't been + * initialized yet. + */ + PROV_SHA2_DIGEST_KEY(mechanism->cm_type / 3, + &sha2_hmac_ctx.hc_icontext, + key->ck_data, keylen_in_bytes, digest); + sha2_mac_init_ctx(&sha2_hmac_ctx, digest, + sha_digest_len); + } else { + sha2_mac_init_ctx(&sha2_hmac_ctx, key->ck_data, + keylen_in_bytes); + } + } + + /* get the mechanism parameters, if applicable */ + if ((mechanism->cm_type % 3) == 2) { + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (ulong_t)) { + ret = CRYPTO_MECHANISM_PARAM_INVALID; + goto bail; + } + PROV_SHA2_GET_DIGEST_LEN(mechanism, digest_len); + if (digest_len > sha_digest_len) { + ret = CRYPTO_MECHANISM_PARAM_INVALID; + goto bail; + } + } + + /* do a SHA2 update of the inner context using the specified data */ + SHA2_MAC_UPDATE(data, sha2_hmac_ctx, ret); + if (ret != CRYPTO_SUCCESS) + /* the update failed, free context and bail */ + goto bail; + + /* + * Do a SHA2 final on the inner context. + */ + SHA2Final(digest, &sha2_hmac_ctx.hc_icontext); + + /* + * Do an SHA2 update on the outer context, feeding the inner + * digest as data. + */ + SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, sha_digest_len); + + /* + * Do a SHA2 final on the outer context, storing the computed + * digest in the users buffer. + */ + switch (mac->cd_format) { + case CRYPTO_DATA_RAW: + if (digest_len != sha_digest_len) { + /* + * The caller requested a short digest. Digest + * into a scratch buffer and return to + * the user only what was requested. + */ + SHA2Final(digest, &sha2_hmac_ctx.hc_ocontext); + bcopy(digest, (unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, digest_len); + } else { + SHA2Final((unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, &sha2_hmac_ctx.hc_ocontext); + } + break; + case CRYPTO_DATA_UIO: + ret = sha2_digest_final_uio(&sha2_hmac_ctx.hc_ocontext, mac, + digest_len, digest); + break; + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + if (ret == CRYPTO_SUCCESS) { + mac->cd_length = digest_len; + return (CRYPTO_SUCCESS); + } +bail: + bzero(&sha2_hmac_ctx, sizeof (sha2_hmac_ctx_t)); + mac->cd_length = 0; + return (ret); +} + +/* ARGSUSED */ +static int +sha2_mac_verify_atomic(crypto_provider_handle_t provider, + crypto_session_id_t session_id, crypto_mechanism_t *mechanism, + crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, + crypto_spi_ctx_template_t ctx_template, crypto_req_handle_t req) +{ + int ret = CRYPTO_SUCCESS; + uchar_t digest[SHA256_DIGEST_LENGTH]; + sha2_hmac_ctx_t sha2_hmac_ctx; + uint32_t sha_digest_len, digest_len, sha_hmac_block_size; + uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length); + + /* + * Set the digest length and block size to values appropriate to the + * mechanism + */ + switch (mechanism->cm_type) { + case SHA256_HMAC_MECH_INFO_TYPE: + case SHA256_HMAC_GEN_MECH_INFO_TYPE: + sha_digest_len = digest_len = SHA256_DIGEST_LENGTH; + sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE; + break; + default: + return (CRYPTO_MECHANISM_INVALID); + } + + /* Add support for key by attributes (RFE 4706552) */ + if (key->ck_format != CRYPTO_KEY_RAW) + return (CRYPTO_ARGUMENTS_BAD); + + if (ctx_template != NULL) { + /* reuse context template */ + bcopy(ctx_template, &sha2_hmac_ctx, sizeof (sha2_hmac_ctx_t)); + } else { + sha2_hmac_ctx.hc_mech_type = mechanism->cm_type; + /* no context template, initialize context */ + if (keylen_in_bytes > sha_hmac_block_size) { + /* + * Hash the passed-in key to get a smaller key. + * The inner context is used since it hasn't been + * initialized yet. + */ + PROV_SHA2_DIGEST_KEY(mechanism->cm_type / 3, + &sha2_hmac_ctx.hc_icontext, + key->ck_data, keylen_in_bytes, digest); + sha2_mac_init_ctx(&sha2_hmac_ctx, digest, + sha_digest_len); + } else { + sha2_mac_init_ctx(&sha2_hmac_ctx, key->ck_data, + keylen_in_bytes); + } + } + + /* get the mechanism parameters, if applicable */ + if (mechanism->cm_type % 3 == 2) { + if (mechanism->cm_param == NULL || + mechanism->cm_param_len != sizeof (ulong_t)) { + ret = CRYPTO_MECHANISM_PARAM_INVALID; + goto bail; + } + PROV_SHA2_GET_DIGEST_LEN(mechanism, digest_len); + if (digest_len > sha_digest_len) { + ret = CRYPTO_MECHANISM_PARAM_INVALID; + goto bail; + } + } + + if (mac->cd_length != digest_len) { + ret = CRYPTO_INVALID_MAC; + goto bail; + } + + /* do a SHA2 update of the inner context using the specified data */ + SHA2_MAC_UPDATE(data, sha2_hmac_ctx, ret); + if (ret != CRYPTO_SUCCESS) + /* the update failed, free context and bail */ + goto bail; + + /* do a SHA2 final on the inner context */ + SHA2Final(digest, &sha2_hmac_ctx.hc_icontext); + + /* + * Do an SHA2 update on the outer context, feeding the inner + * digest as data. + */ + SHA2Update(&sha2_hmac_ctx.hc_ocontext, digest, sha_digest_len); + + /* + * Do a SHA2 final on the outer context, storing the computed + * digest in the users buffer. + */ + SHA2Final(digest, &sha2_hmac_ctx.hc_ocontext); + + /* + * Compare the computed digest against the expected digest passed + * as argument. + */ + + switch (mac->cd_format) { + + case CRYPTO_DATA_RAW: + if (bcmp(digest, (unsigned char *)mac->cd_raw.iov_base + + mac->cd_offset, digest_len) != 0) + ret = CRYPTO_INVALID_MAC; + break; + + case CRYPTO_DATA_UIO: { + off_t offset = mac->cd_offset; + uint_t vec_idx; + off_t scratch_offset = 0; + size_t length = digest_len; + size_t cur_len; + + /* we support only kernel buffer */ + if (mac->cd_uio->uio_segflg != UIO_SYSSPACE) + return (CRYPTO_ARGUMENTS_BAD); + + /* jump to the first iovec containing the expected digest */ + for (vec_idx = 0; + offset >= mac->cd_uio->uio_iov[vec_idx].iov_len && + vec_idx < mac->cd_uio->uio_iovcnt; + offset -= mac->cd_uio->uio_iov[vec_idx++].iov_len) + ; + if (vec_idx == mac->cd_uio->uio_iovcnt) { + /* + * The caller specified an offset that is + * larger than the total size of the buffers + * it provided. + */ + ret = CRYPTO_DATA_LEN_RANGE; + break; + } + + /* do the comparison of computed digest vs specified one */ + while (vec_idx < mac->cd_uio->uio_iovcnt && length > 0) { + cur_len = MIN(mac->cd_uio->uio_iov[vec_idx].iov_len - + offset, length); + + if (bcmp(digest + scratch_offset, + mac->cd_uio->uio_iov[vec_idx].iov_base + offset, + cur_len) != 0) { + ret = CRYPTO_INVALID_MAC; + break; + } + + length -= cur_len; + vec_idx++; + scratch_offset += cur_len; + offset = 0; + } + break; + } + + default: + ret = CRYPTO_ARGUMENTS_BAD; + } + + return (ret); +bail: + bzero(&sha2_hmac_ctx, sizeof (sha2_hmac_ctx_t)); + mac->cd_length = 0; + return (ret); +} + +/* + * KCF software provider context management entry points. + */ + +/* ARGSUSED */ +static int +sha2_create_ctx_template(crypto_provider_handle_t provider, + crypto_mechanism_t *mechanism, crypto_key_t *key, + crypto_spi_ctx_template_t *ctx_template, size_t *ctx_template_size, + crypto_req_handle_t req) +{ + sha2_hmac_ctx_t *sha2_hmac_ctx_tmpl; + uint_t keylen_in_bytes = CRYPTO_BITS2BYTES(key->ck_length); + uint32_t sha_digest_len, sha_hmac_block_size; + + /* + * Set the digest length and block size to values appropriate to the + * mechanism + */ + switch (mechanism->cm_type) { + case SHA256_HMAC_MECH_INFO_TYPE: + case SHA256_HMAC_GEN_MECH_INFO_TYPE: + sha_digest_len = SHA256_DIGEST_LENGTH; + sha_hmac_block_size = SHA256_HMAC_BLOCK_SIZE; + break; + default: + return (CRYPTO_MECHANISM_INVALID); + } + + /* Add support for key by attributes (RFE 4706552) */ + if (key->ck_format != CRYPTO_KEY_RAW) + return (CRYPTO_ARGUMENTS_BAD); + + /* + * Allocate and initialize SHA2 context. + */ + sha2_hmac_ctx_tmpl = kmem_alloc(sizeof (sha2_hmac_ctx_t), + crypto_kmflag(req)); + if (sha2_hmac_ctx_tmpl == NULL) + return (CRYPTO_HOST_MEMORY); + + sha2_hmac_ctx_tmpl->hc_mech_type = mechanism->cm_type; + + if (keylen_in_bytes > sha_hmac_block_size) { + uchar_t digested_key[SHA256_DIGEST_LENGTH]; + + /* + * Hash the passed-in key to get a smaller key. + * The inner context is used since it hasn't been + * initialized yet. + */ + PROV_SHA2_DIGEST_KEY(mechanism->cm_type / 3, + &sha2_hmac_ctx_tmpl->hc_icontext, + key->ck_data, keylen_in_bytes, digested_key); + sha2_mac_init_ctx(sha2_hmac_ctx_tmpl, digested_key, + sha_digest_len); + } else { + sha2_mac_init_ctx(sha2_hmac_ctx_tmpl, key->ck_data, + keylen_in_bytes); + } + + *ctx_template = (crypto_spi_ctx_template_t)sha2_hmac_ctx_tmpl; + *ctx_template_size = sizeof (sha2_hmac_ctx_t); + + return (CRYPTO_SUCCESS); +} + +static int +sha2_free_context(crypto_ctx_t *ctx) +{ + uint_t ctx_len; + + if (ctx->cc_provider_private == NULL) + return (CRYPTO_SUCCESS); + + /* + * We have to free either SHA2 or SHA2-HMAC contexts, which + * have different lengths. + * + * Note: Below is dependent on the mechanism ordering. + */ + + if (PROV_SHA2_CTX(ctx)->sc_mech_type % 3 == 0) + ctx_len = sizeof (sha2_ctx_t); + else + ctx_len = sizeof (sha2_hmac_ctx_t); + + bzero(ctx->cc_provider_private, ctx_len); + kmem_free(ctx->cc_provider_private, ctx_len); + ctx->cc_provider_private = NULL; + + return (CRYPTO_SUCCESS); +} diff --git a/module/icp/os/modconf.c b/module/icp/os/modconf.c new file mode 100644 index 000000000000..e0cd7f4ad876 --- /dev/null +++ b/module/icp/os/modconf.c @@ -0,0 +1,171 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#include +#include + +/* + * Null operations; used for uninitialized and "misc" modules. + */ +static int mod_null(struct modlmisc *, struct modlinkage *); +static int mod_infonull(void *, struct modlinkage *, int *); + +/* + * Cryptographic Modules + */ +struct mod_ops mod_cryptoops = { + mod_null, mod_null, mod_infonull +}; + +/* + * Null operation; return 0. + */ +static int +mod_null(struct modlmisc *modl, struct modlinkage *modlp) +{ + return (0); +} + +/* + * Status for User modules. + */ +static int +mod_infonull(void *modl, struct modlinkage *modlp, int *p0) +{ + *p0 = -1; /* for modinfo display */ + return (0); +} + +/* + * Install a module. + * (This routine is in the Solaris SPARC DDI/DKI) + */ +int +mod_install(struct modlinkage *modlp) +{ + int retval = -1; /* No linkage structures */ + struct modlmisc **linkpp; + struct modlmisc **linkpp1; + + if (modlp->ml_rev != MODREV_1) { + cmn_err(CE_WARN, "mod_install: " + "modlinkage structure is not MODREV_1\n"); + return (EINVAL); + } + linkpp = (struct modlmisc **)&modlp->ml_linkage[0]; + + while (*linkpp != NULL) { + if ((retval = MODL_INSTALL(*linkpp, modlp)) != 0) { + linkpp1 = (struct modlmisc **)&modlp->ml_linkage[0]; + + while (linkpp1 != linkpp) { + MODL_REMOVE(*linkpp1, modlp); /* clean up */ + linkpp1++; + } + break; + } + linkpp++; + } + return (retval); +} + +static char *reins_err = + "Could not reinstall %s\nReboot to correct the problem"; + +/* + * Remove a module. This is called by the module wrapper routine. + * (This routine is in the Solaris SPARC DDI/DKI) + */ +int +mod_remove(struct modlinkage *modlp) +{ + int retval = 0; + struct modlmisc **linkpp, *last_linkp; + + linkpp = (struct modlmisc **)&modlp->ml_linkage[0]; + + while (*linkpp != NULL) { + if ((retval = MODL_REMOVE(*linkpp, modlp)) != 0) { + last_linkp = *linkpp; + linkpp = (struct modlmisc **)&modlp->ml_linkage[0]; + while (*linkpp != last_linkp) { + if (MODL_INSTALL(*linkpp, modlp) != 0) { + cmn_err(CE_WARN, reins_err, + (*linkpp)->misc_linkinfo); + break; + } + linkpp++; + } + break; + } + linkpp++; + } + return (retval); +} + +/* + * Get module status. + * (This routine is in the Solaris SPARC DDI/DKI) + */ +int +mod_info(struct modlinkage *modlp, struct modinfo *modinfop) +{ + int i; + int retval = 0; + struct modspecific_info *msip; + struct modlmisc **linkpp; + + modinfop->mi_rev = modlp->ml_rev; + + linkpp = (struct modlmisc **)modlp->ml_linkage; + msip = &modinfop->mi_msinfo[0]; + + for (i = 0; i < MODMAXLINK; i++) { + if (*linkpp == NULL) { + msip->msi_linkinfo[0] = '\0'; + } else { + (void) strncpy(msip->msi_linkinfo, + (*linkpp)->misc_linkinfo, MODMAXLINKINFOLEN); + retval = MODL_INFO(*linkpp, modlp, &msip->msi_p0); + if (retval != 0) + break; + linkpp++; + } + msip++; + } + + if (modinfop->mi_info == MI_INFO_LINKAGE) { + /* + * Slight kludge used to extract the address of the + * modlinkage structure from the module (just after + * loading a module for the very first time) + */ + modinfop->mi_base = (void *)modlp; + } + + if (retval == 0) + return (1); + return (0); +} \ No newline at end of file diff --git a/module/icp/os/modhash.c b/module/icp/os/modhash.c new file mode 100644 index 000000000000..1ff782afc0ce --- /dev/null +++ b/module/icp/os/modhash.c @@ -0,0 +1,925 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * mod_hash: flexible hash table implementation. + * + * This is a reasonably fast, reasonably flexible hash table implementation + * which features pluggable hash algorithms to support storing arbitrary keys + * and values. It is designed to handle small (< 100,000 items) amounts of + * data. The hash uses chaining to resolve collisions, and does not feature a + * mechanism to grow the hash. Care must be taken to pick nchains to be large + * enough for the application at hand, or lots of time will be wasted searching + * hash chains. + * + * The client of the hash is required to supply a number of items to support + * the various hash functions: + * + * - Destructor functions for the key and value being hashed. + * A destructor is responsible for freeing an object when the hash + * table is no longer storing it. Since keys and values can be of + * arbitrary type, separate destructors for keys & values are used. + * These may be mod_hash_null_keydtor and mod_hash_null_valdtor if no + * destructor is needed for either a key or value. + * + * - A hashing algorithm which returns a uint_t representing a hash index + * The number returned need _not_ be between 0 and nchains. The mod_hash + * code will take care of doing that. The second argument (after the + * key) to the hashing function is a void * that represents + * hash_alg_data-- this is provided so that the hashing algrorithm can + * maintain some state across calls, or keep algorithm-specific + * constants associated with the hash table. + * + * A pointer-hashing and a string-hashing algorithm are supplied in + * this file. + * + * - A key comparator (a la qsort). + * This is used when searching the hash chain. The key comparator + * determines if two keys match. It should follow the return value + * semantics of strcmp. + * + * string and pointer comparators are supplied in this file. + * + * mod_hash_create_strhash() and mod_hash_create_ptrhash() provide good + * examples of how to create a customized hash table. + * + * Basic hash operations: + * + * mod_hash_create_strhash(name, nchains, dtor), + * create a hash using strings as keys. + * NOTE: This create a hash which automatically cleans up the string + * values it is given for keys. + * + * mod_hash_create_ptrhash(name, nchains, dtor, key_elem_size): + * create a hash using pointers as keys. + * + * mod_hash_create_extended(name, nchains, kdtor, vdtor, + * hash_alg, hash_alg_data, + * keycmp, sleep) + * create a customized hash table. + * + * mod_hash_destroy_hash(hash): + * destroy the given hash table, calling the key and value destructors + * on each key-value pair stored in the hash. + * + * mod_hash_insert(hash, key, val): + * place a key, value pair into the given hash. + * duplicate keys are rejected. + * + * mod_hash_insert_reserve(hash, key, val, handle): + * place a key, value pair into the given hash, using handle to indicate + * the reserved storage for the pair. (no memory allocation is needed + * during a mod_hash_insert_reserve.) duplicate keys are rejected. + * + * mod_hash_reserve(hash, *handle): + * reserve storage for a key-value pair using the memory allocation + * policy of 'hash', returning the storage handle in 'handle'. + * + * mod_hash_reserve_nosleep(hash, *handle): reserve storage for a key-value + * pair ignoring the memory allocation policy of 'hash' and always without + * sleep, returning the storage handle in 'handle'. + * + * mod_hash_remove(hash, key, *val): + * remove a key-value pair with key 'key' from 'hash', destroying the + * stored key, and returning the value in val. + * + * mod_hash_replace(hash, key, val) + * atomically remove an existing key-value pair from a hash, and replace + * the key and value with the ones supplied. The removed key and value + * (if any) are destroyed. + * + * mod_hash_destroy(hash, key): + * remove a key-value pair with key 'key' from 'hash', destroying both + * stored key and stored value. + * + * mod_hash_find(hash, key, val): + * find a value in the hash table corresponding to the given key. + * + * mod_hash_find_cb(hash, key, val, found_callback) + * find a value in the hash table corresponding to the given key. + * If a value is found, call specified callback passing key and val to it. + * The callback is called with the hash lock held. + * It is intended to be used in situations where the act of locating the + * data must also modify it - such as in reference counting schemes. + * + * mod_hash_walk(hash, callback(key, elem, arg), arg) + * walks all the elements in the hashtable and invokes the callback + * function with the key/value pair for each element. the hashtable + * is locked for readers so the callback function should not attempt + * to do any updates to the hashable. the callback function should + * return MH_WALK_CONTINUE to continue walking the hashtable or + * MH_WALK_TERMINATE to abort the walk of the hashtable. + * + * mod_hash_clear(hash): + * clears the given hash table of entries, calling the key and value + * destructors for every element in the hash. + */ + +#include +#include +#include +#include + +/* + * MH_KEY_DESTROY() + * Invoke the key destructor. + */ +#define MH_KEY_DESTROY(hash, key) ((hash->mh_kdtor)(key)) + +/* + * MH_VAL_DESTROY() + * Invoke the value destructor. + */ +#define MH_VAL_DESTROY(hash, val) ((hash->mh_vdtor)(val)) + +/* + * MH_KEYCMP() + * Call the key comparator for the given hash keys. + */ +#define MH_KEYCMP(hash, key1, key2) ((hash->mh_keycmp)(key1, key2)) + +/* + * Cache for struct mod_hash_entry + */ +kmem_cache_t *mh_e_cache = NULL; +mod_hash_t *mh_head = NULL; +kmutex_t mh_head_lock; + +/* + * mod_hash_null_keydtor() + * mod_hash_null_valdtor() + * no-op key and value destructors. + */ +/*ARGSUSED*/ +void +mod_hash_null_keydtor(mod_hash_key_t key) +{ +} + +/*ARGSUSED*/ +void +mod_hash_null_valdtor(mod_hash_val_t val) +{ +} + +/* + * mod_hash_bystr() + * mod_hash_strkey_cmp() + * mod_hash_strkey_dtor() + * mod_hash_strval_dtor() + * Hash and key comparison routines for hashes with string keys. + * + * mod_hash_create_strhash() + * Create a hash using strings as keys + * + * The string hashing algorithm is from the "Dragon Book" -- + * "Compilers: Principles, Tools & Techniques", by Aho, Sethi, Ullman + */ + +/*ARGSUSED*/ +uint_t +mod_hash_bystr(void *hash_data, mod_hash_key_t key) +{ + uint_t hash = 0; + uint_t g; + char *p, *k = (char *)key; + + ASSERT(k); + for (p = k; *p != '\0'; p++) { + hash = (hash << 4) + *p; + if ((g = (hash & 0xf0000000)) != 0) { + hash ^= (g >> 24); + hash ^= g; + } + } + return (hash); +} + +int +mod_hash_strkey_cmp(mod_hash_key_t key1, mod_hash_key_t key2) +{ + return (strcmp((char *)key1, (char *)key2)); +} + +void +mod_hash_strkey_dtor(mod_hash_key_t key) +{ + char *c = (char *)key; + kmem_free(c, strlen(c) + 1); +} + +void +mod_hash_strval_dtor(mod_hash_val_t val) +{ + char *c = (char *)val; + kmem_free(c, strlen(c) + 1); +} + +mod_hash_t * +mod_hash_create_strhash_nodtr(char *name, size_t nchains, + void (*val_dtor)(mod_hash_val_t)) +{ + return mod_hash_create_extended(name, nchains, mod_hash_null_keydtor, + val_dtor, mod_hash_bystr, NULL, mod_hash_strkey_cmp, KM_SLEEP); +} + +mod_hash_t * +mod_hash_create_strhash(char *name, size_t nchains, + void (*val_dtor)(mod_hash_val_t)) +{ + return mod_hash_create_extended(name, nchains, mod_hash_strkey_dtor, + val_dtor, mod_hash_bystr, NULL, mod_hash_strkey_cmp, KM_SLEEP); +} + +void +mod_hash_destroy_strhash(mod_hash_t *strhash) +{ + ASSERT(strhash); + mod_hash_destroy_hash(strhash); +} + + +/* + * mod_hash_byptr() + * mod_hash_ptrkey_cmp() + * Hash and key comparison routines for hashes with pointer keys. + * + * mod_hash_create_ptrhash() + * mod_hash_destroy_ptrhash() + * Create a hash that uses pointers as keys. This hash algorithm + * picks an appropriate set of middle bits in the address to hash on + * based on the size of the hash table and a hint about the size of + * the items pointed at. + */ +uint_t +mod_hash_byptr(void *hash_data, mod_hash_key_t key) +{ + uintptr_t k = (uintptr_t)key; + k >>= (int)(uintptr_t)hash_data; + + return ((uint_t)k); +} + +int +mod_hash_ptrkey_cmp(mod_hash_key_t key1, mod_hash_key_t key2) +{ + uintptr_t k1 = (uintptr_t)key1; + uintptr_t k2 = (uintptr_t)key2; + if (k1 > k2) + return (-1); + else if (k1 < k2) + return (1); + else + return (0); +} + +mod_hash_t * +mod_hash_create_ptrhash(char *name, size_t nchains, + void (*val_dtor)(mod_hash_val_t), size_t key_elem_size) +{ + size_t rshift; + + /* + * We want to hash on the bits in the middle of the address word + * Bits far to the right in the word have little significance, and + * are likely to all look the same (for example, an array of + * 256-byte structures will have the bottom 8 bits of address + * words the same). So we want to right-shift each address to + * ignore the bottom bits. + * + * The high bits, which are also unused, will get taken out when + * mod_hash takes hashkey % nchains. + */ + rshift = highbit(key_elem_size); + + return mod_hash_create_extended(name, nchains, mod_hash_null_keydtor, + val_dtor, mod_hash_byptr, (void *)rshift, mod_hash_ptrkey_cmp, + KM_SLEEP); +} + +void +mod_hash_destroy_ptrhash(mod_hash_t *hash) +{ + ASSERT(hash); + mod_hash_destroy_hash(hash); +} + +/* + * mod_hash_byid() + * mod_hash_idkey_cmp() + * Hash and key comparison routines for hashes with 32-bit unsigned keys. + * + * mod_hash_create_idhash() + * mod_hash_destroy_idhash() + * mod_hash_iddata_gen() + * Create a hash that uses numeric keys. + * + * The hash algorithm is documented in "Introduction to Algorithms" + * (Cormen, Leiserson, Rivest); when the hash table is created, it + * attempts to find the next largest prime above the number of hash + * slots. The hash index is then this number times the key modulo + * the hash size, or (key * prime) % nchains. + */ +uint_t +mod_hash_byid(void *hash_data, mod_hash_key_t key) +{ + uint_t kval = (uint_t)(uintptr_t)hash_data; + return ((uint_t)(uintptr_t)key * (uint_t)kval); +} + +int +mod_hash_idkey_cmp(mod_hash_key_t key1, mod_hash_key_t key2) +{ + return ((uint_t)(uintptr_t)key1 - (uint_t)(uintptr_t)key2); +} + +/* + * Generate the next largest prime number greater than nchains; this value + * is intended to be later passed in to mod_hash_create_extended() as the + * hash_data. + */ +uint_t +mod_hash_iddata_gen(size_t nchains) +{ + uint_t kval, i, prime; + + /* + * Pick the first (odd) prime greater than nchains. Make sure kval is + * odd (so start with nchains +1 or +2 as appropriate). + */ + kval = (nchains % 2 == 0) ? nchains + 1 : nchains + 2; + + for (;;) { + prime = 1; + for (i = 3; i * i <= kval; i += 2) { + if (kval % i == 0) + prime = 0; + } + if (prime == 1) + break; + kval += 2; + } + return (kval); +} + +mod_hash_t * +mod_hash_create_idhash(char *name, size_t nchains, + void (*val_dtor)(mod_hash_val_t)) +{ + uint_t kval = mod_hash_iddata_gen(nchains); + + return (mod_hash_create_extended(name, nchains, mod_hash_null_keydtor, + val_dtor, mod_hash_byid, (void *)(uintptr_t)kval, + mod_hash_idkey_cmp, KM_SLEEP)); +} + +void +mod_hash_destroy_idhash(mod_hash_t *hash) +{ + ASSERT(hash); + mod_hash_destroy_hash(hash); +} + +void +mod_hash_fini(void) +{ + mutex_destroy(&mh_head_lock); + + if (mh_e_cache) { + kmem_cache_destroy(mh_e_cache); + mh_e_cache = NULL; + } +} + +/* + * mod_hash_init() + * sets up globals, etc for mod_hash_* + */ +void +mod_hash_init(void) +{ + ASSERT(mh_e_cache == NULL); + mh_e_cache = kmem_cache_create("mod_hash_entries", + sizeof (struct mod_hash_entry), 0, NULL, NULL, NULL, NULL, + NULL, 0); + + mutex_init(&mh_head_lock, NULL, MUTEX_DEFAULT, NULL); +} + +/* + * mod_hash_create_extended() + * The full-blown hash creation function. + * + * notes: + * nchains - how many hash slots to create. More hash slots will + * result in shorter hash chains, but will consume + * slightly more memory up front. + * sleep - should be KM_SLEEP or KM_NOSLEEP, to indicate whether + * to sleep for memory, or fail in low-memory conditions. + * + * Fails only if KM_NOSLEEP was specified, and no memory was available. + */ +mod_hash_t * +mod_hash_create_extended( + char *hname, /* descriptive name for hash */ + size_t nchains, /* number of hash slots */ + void (*kdtor)(mod_hash_key_t), /* key destructor */ + void (*vdtor)(mod_hash_val_t), /* value destructor */ + uint_t (*hash_alg)(void *, mod_hash_key_t), /* hash algorithm */ + void *hash_alg_data, /* pass-thru arg for hash_alg */ + int (*keycmp)(mod_hash_key_t, mod_hash_key_t), /* key comparator */ + int sleep) /* whether to sleep for mem */ +{ + mod_hash_t *mod_hash; + ASSERT(hname && keycmp && hash_alg && vdtor && kdtor); + + if ((mod_hash = kmem_zalloc(MH_SIZE(nchains), sleep)) == NULL) + return (NULL); + + mod_hash->mh_name = kmem_alloc(strlen(hname) + 1, sleep); + if (mod_hash->mh_name == NULL) { + kmem_free(mod_hash, MH_SIZE(nchains)); + return (NULL); + } + (void) strcpy(mod_hash->mh_name, hname); + + rw_init(&mod_hash->mh_contents, NULL, RW_DEFAULT, NULL); + mod_hash->mh_sleep = sleep; + mod_hash->mh_nchains = nchains; + mod_hash->mh_kdtor = kdtor; + mod_hash->mh_vdtor = vdtor; + mod_hash->mh_hashalg = hash_alg; + mod_hash->mh_hashalg_data = hash_alg_data; + mod_hash->mh_keycmp = keycmp; + + /* + * Link the hash up on the list of hashes + */ + mutex_enter(&mh_head_lock); + mod_hash->mh_next = mh_head; + mh_head = mod_hash; + mutex_exit(&mh_head_lock); + + return (mod_hash); +} + +/* + * mod_hash_destroy_hash() + * destroy a hash table, destroying all of its stored keys and values + * as well. + */ +void +mod_hash_destroy_hash(mod_hash_t *hash) +{ + mod_hash_t *mhp, *mhpp; + + mutex_enter(&mh_head_lock); + /* + * Remove the hash from the hash list + */ + if (hash == mh_head) { /* removing 1st list elem */ + mh_head = mh_head->mh_next; + } else { + /* + * mhpp can start out NULL since we know the 1st elem isn't the + * droid we're looking for. + */ + mhpp = NULL; + for (mhp = mh_head; mhp != NULL; mhp = mhp->mh_next) { + if (mhp == hash) { + mhpp->mh_next = mhp->mh_next; + break; + } + mhpp = mhp; + } + } + mutex_exit(&mh_head_lock); + + /* + * Clean out keys and values. + */ + mod_hash_clear(hash); + + rw_destroy(&hash->mh_contents); + kmem_free(hash->mh_name, strlen(hash->mh_name) + 1); + kmem_free(hash, MH_SIZE(hash->mh_nchains)); +} + +/* + * i_mod_hash() + * Call the hashing algorithm for this hash table, with the given key. + */ +uint_t +i_mod_hash(mod_hash_t *hash, mod_hash_key_t key) +{ + uint_t h; + /* + * Prevent div by 0 problems; + * Also a nice shortcut when using a hash as a list + */ + if (hash->mh_nchains == 1) + return (0); + + h = (hash->mh_hashalg)(hash->mh_hashalg_data, key); + return (h % (hash->mh_nchains - 1)); +} + +/* + * i_mod_hash_insert_nosync() + * mod_hash_insert() + * mod_hash_insert_reserve() + * insert 'val' into the hash table, using 'key' as its key. If 'key' is + * already a key in the hash, an error will be returned, and the key-val + * pair will not be inserted. i_mod_hash_insert_nosync() supports a simple + * handle abstraction, allowing hash entry allocation to be separated from + * the hash insertion. this abstraction allows simple use of the mod_hash + * structure in situations where mod_hash_insert() with a KM_SLEEP + * allocation policy would otherwise be unsafe. + */ +int +i_mod_hash_insert_nosync(mod_hash_t *hash, mod_hash_key_t key, + mod_hash_val_t val, mod_hash_hndl_t handle) +{ + uint_t hashidx; + struct mod_hash_entry *entry; + + ASSERT(hash); + + /* + * If we've not been given reserved storage, allocate storage directly, + * using the hash's allocation policy. + */ + if (handle == (mod_hash_hndl_t)0) { + entry = kmem_cache_alloc(mh_e_cache, hash->mh_sleep); + if (entry == NULL) { + hash->mh_stat.mhs_nomem++; + return (MH_ERR_NOMEM); + } + } else { + entry = (struct mod_hash_entry *)handle; + } + + hashidx = i_mod_hash(hash, key); + entry->mhe_key = key; + entry->mhe_val = val; + entry->mhe_next = hash->mh_entries[hashidx]; + + hash->mh_entries[hashidx] = entry; + hash->mh_stat.mhs_nelems++; + + return (0); +} + +int +mod_hash_insert(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t val) +{ + int res; + mod_hash_val_t v; + + rw_enter(&hash->mh_contents, RW_WRITER); + + /* + * Disallow duplicate keys in the hash + */ + if (i_mod_hash_find_nosync(hash, key, &v) == 0) { + rw_exit(&hash->mh_contents); + hash->mh_stat.mhs_coll++; + return (MH_ERR_DUPLICATE); + } + + res = i_mod_hash_insert_nosync(hash, key, val, (mod_hash_hndl_t)0); + rw_exit(&hash->mh_contents); + + return (res); +} + +int +mod_hash_insert_reserve(mod_hash_t *hash, mod_hash_key_t key, + mod_hash_val_t val, mod_hash_hndl_t handle) +{ + int res; + mod_hash_val_t v; + + rw_enter(&hash->mh_contents, RW_WRITER); + + /* + * Disallow duplicate keys in the hash + */ + if (i_mod_hash_find_nosync(hash, key, &v) == 0) { + rw_exit(&hash->mh_contents); + hash->mh_stat.mhs_coll++; + return (MH_ERR_DUPLICATE); + } + res = i_mod_hash_insert_nosync(hash, key, val, handle); + rw_exit(&hash->mh_contents); + + return (res); +} + +/* + * mod_hash_reserve() + * mod_hash_reserve_nosleep() + * mod_hash_cancel() + * Make or cancel a mod_hash_entry_t reservation. Reservations are used in + * mod_hash_insert_reserve() above. + */ +int +mod_hash_reserve(mod_hash_t *hash, mod_hash_hndl_t *handlep) +{ + *handlep = kmem_cache_alloc(mh_e_cache, hash->mh_sleep); + if (*handlep == NULL) { + hash->mh_stat.mhs_nomem++; + return (MH_ERR_NOMEM); + } + + return (0); +} + +int +mod_hash_reserve_nosleep(mod_hash_t *hash, mod_hash_hndl_t *handlep) +{ + *handlep = kmem_cache_alloc(mh_e_cache, KM_NOSLEEP); + if (*handlep == NULL) { + hash->mh_stat.mhs_nomem++; + return (MH_ERR_NOMEM); + } + + return (0); + +} + +/*ARGSUSED*/ +void +mod_hash_cancel(mod_hash_t *hash, mod_hash_hndl_t *handlep) +{ + kmem_cache_free(mh_e_cache, *handlep); + *handlep = (mod_hash_hndl_t)0; +} + +/* + * i_mod_hash_remove_nosync() + * mod_hash_remove() + * Remove an element from the hash table. + */ +int +i_mod_hash_remove_nosync(mod_hash_t *hash, mod_hash_key_t key, + mod_hash_val_t *val) +{ + int hashidx; + struct mod_hash_entry *e, *ep; + + hashidx = i_mod_hash(hash, key); + ep = NULL; /* e's parent */ + + for (e = hash->mh_entries[hashidx]; e != NULL; e = e->mhe_next) { + if (MH_KEYCMP(hash, e->mhe_key, key) == 0) + break; + ep = e; + } + + if (e == NULL) { /* not found */ + return (MH_ERR_NOTFOUND); + } + + if (ep == NULL) /* special case 1st element in bucket */ + hash->mh_entries[hashidx] = e->mhe_next; + else + ep->mhe_next = e->mhe_next; + + /* + * Clean up resources used by the node's key. + */ + MH_KEY_DESTROY(hash, e->mhe_key); + + *val = e->mhe_val; + kmem_cache_free(mh_e_cache, e); + hash->mh_stat.mhs_nelems--; + + return (0); +} + +int +mod_hash_remove(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t *val) +{ + int res; + + rw_enter(&hash->mh_contents, RW_WRITER); + res = i_mod_hash_remove_nosync(hash, key, val); + rw_exit(&hash->mh_contents); + + return (res); +} + +/* + * mod_hash_replace() + * atomically remove an existing key-value pair from a hash, and replace + * the key and value with the ones supplied. The removed key and value + * (if any) are destroyed. + */ +int +mod_hash_replace(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t val) +{ + int res; + mod_hash_val_t v; + + rw_enter(&hash->mh_contents, RW_WRITER); + + if (i_mod_hash_remove_nosync(hash, key, &v) == 0) { + /* + * mod_hash_remove() takes care of freeing up the key resources. + */ + MH_VAL_DESTROY(hash, v); + } + res = i_mod_hash_insert_nosync(hash, key, val, (mod_hash_hndl_t)0); + + rw_exit(&hash->mh_contents); + + return (res); +} + +/* + * mod_hash_destroy() + * Remove an element from the hash table matching 'key', and destroy it. + */ +int +mod_hash_destroy(mod_hash_t *hash, mod_hash_key_t key) +{ + mod_hash_val_t val; + int rv; + + rw_enter(&hash->mh_contents, RW_WRITER); + + if ((rv = i_mod_hash_remove_nosync(hash, key, &val)) == 0) { + /* + * mod_hash_remove() takes care of freeing up the key resources. + */ + MH_VAL_DESTROY(hash, val); + } + + rw_exit(&hash->mh_contents); + return (rv); +} + +/* + * i_mod_hash_find_nosync() + * mod_hash_find() + * Find a value in the hash table corresponding to the given key. + */ +int +i_mod_hash_find_nosync(mod_hash_t *hash, mod_hash_key_t key, + mod_hash_val_t *val) +{ + uint_t hashidx; + struct mod_hash_entry *e; + + hashidx = i_mod_hash(hash, key); + + for (e = hash->mh_entries[hashidx]; e != NULL; e = e->mhe_next) { + if (MH_KEYCMP(hash, e->mhe_key, key) == 0) { + *val = e->mhe_val; + hash->mh_stat.mhs_hit++; + return (0); + } + } + hash->mh_stat.mhs_miss++; + return (MH_ERR_NOTFOUND); +} + +int +mod_hash_find(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t *val) +{ + int res; + + rw_enter(&hash->mh_contents, RW_READER); + res = i_mod_hash_find_nosync(hash, key, val); + rw_exit(&hash->mh_contents); + + return (res); +} + +int +mod_hash_find_cb(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t *val, + void (*find_cb)(mod_hash_key_t, mod_hash_val_t)) +{ + int res; + + rw_enter(&hash->mh_contents, RW_READER); + res = i_mod_hash_find_nosync(hash, key, val); + if (res == 0) { + find_cb(key, *val); + } + rw_exit(&hash->mh_contents); + + return (res); +} + +int +mod_hash_find_cb_rval(mod_hash_t *hash, mod_hash_key_t key, mod_hash_val_t *val, + int (*find_cb)(mod_hash_key_t, mod_hash_val_t), int *cb_rval) +{ + int res; + + rw_enter(&hash->mh_contents, RW_READER); + res = i_mod_hash_find_nosync(hash, key, val); + if (res == 0) { + *cb_rval = find_cb(key, *val); + } + rw_exit(&hash->mh_contents); + + return (res); +} + +void +i_mod_hash_walk_nosync(mod_hash_t *hash, + uint_t (*callback)(mod_hash_key_t, mod_hash_val_t *, void *), void *arg) +{ + struct mod_hash_entry *e; + uint_t hashidx; + int res = MH_WALK_CONTINUE; + + for (hashidx = 0; + (hashidx < (hash->mh_nchains - 1)) && (res == MH_WALK_CONTINUE); + hashidx++) { + e = hash->mh_entries[hashidx]; + while ((e != NULL) && (res == MH_WALK_CONTINUE)) { + res = callback(e->mhe_key, e->mhe_val, arg); + e = e->mhe_next; + } + } +} + +/* + * mod_hash_walk() + * Walks all the elements in the hashtable and invokes the callback + * function with the key/value pair for each element. The hashtable + * is locked for readers so the callback function should not attempt + * to do any updates to the hashable. The callback function should + * return MH_WALK_CONTINUE to continue walking the hashtable or + * MH_WALK_TERMINATE to abort the walk of the hashtable. + */ +void +mod_hash_walk(mod_hash_t *hash, + uint_t (*callback)(mod_hash_key_t, mod_hash_val_t *, void *), void *arg) +{ + rw_enter(&hash->mh_contents, RW_READER); + i_mod_hash_walk_nosync(hash, callback, arg); + rw_exit(&hash->mh_contents); +} + + +/* + * i_mod_hash_clear_nosync() + * mod_hash_clear() + * Clears the given hash table by calling the destructor of every hash + * element and freeing up all mod_hash_entry's. + */ +void +i_mod_hash_clear_nosync(mod_hash_t *hash) +{ + int i; + struct mod_hash_entry *e, *old_e; + + for (i = 0; i < hash->mh_nchains; i++) { + e = hash->mh_entries[i]; + while (e != NULL) { + MH_KEY_DESTROY(hash, e->mhe_key); + MH_VAL_DESTROY(hash, e->mhe_val); + old_e = e; + e = e->mhe_next; + kmem_cache_free(mh_e_cache, old_e); + } + hash->mh_entries[i] = NULL; + } + hash->mh_stat.mhs_nelems = 0; +} + +void +mod_hash_clear(mod_hash_t *hash) +{ + ASSERT(hash); + rw_enter(&hash->mh_contents, RW_WRITER); + i_mod_hash_clear_nosync(hash); + rw_exit(&hash->mh_contents); +} diff --git a/module/icp/spi/kcf_spi.c b/module/icp/spi/kcf_spi.c new file mode 100644 index 000000000000..e6e463a62848 --- /dev/null +++ b/module/icp/spi/kcf_spi.c @@ -0,0 +1,927 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +/* + * This file is part of the core Kernel Cryptographic Framework. + * It implements the SPI functions exported to cryptographic + * providers. + */ + + +#include +#include +#include +#include +#include + +/* + * minalloc and maxalloc values to be used for taskq_create(). + */ +int crypto_taskq_threads = CRYPTO_TASKQ_THREADS; +int crypto_taskq_minalloc = CYRPTO_TASKQ_MIN; +int crypto_taskq_maxalloc = CRYPTO_TASKQ_MAX; + +static void remove_provider(kcf_provider_desc_t *); +static void process_logical_providers(crypto_provider_info_t *, + kcf_provider_desc_t *); +static int init_prov_mechs(crypto_provider_info_t *, kcf_provider_desc_t *); +static int kcf_prov_kstat_update(kstat_t *, int); +static void delete_kstat(kcf_provider_desc_t *); + +static kcf_prov_stats_t kcf_stats_ks_data_template = { + { "kcf_ops_total", KSTAT_DATA_UINT64 }, + { "kcf_ops_passed", KSTAT_DATA_UINT64 }, + { "kcf_ops_failed", KSTAT_DATA_UINT64 }, + { "kcf_ops_returned_busy", KSTAT_DATA_UINT64 } +}; + +#define KCF_SPI_COPY_OPS(src, dst, ops) if ((src)->ops != NULL) \ + *((dst)->ops) = *((src)->ops); + +/* + * Copy an ops vector from src to dst. Used during provider registration + * to copy the ops vector from the provider info structure to the + * provider descriptor maintained by KCF. + * Copying the ops vector specified by the provider is needed since the + * framework does not require the provider info structure to be + * persistent. + */ +static void +copy_ops_vector_v1(crypto_ops_t *src_ops, crypto_ops_t *dst_ops) +{ + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_control_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_digest_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_cipher_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_mac_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_sign_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_verify_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_dual_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_dual_cipher_mac_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_random_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_session_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_object_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_key_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_provider_ops); + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_ctx_ops); +} + +static void +copy_ops_vector_v2(crypto_ops_t *src_ops, crypto_ops_t *dst_ops) +{ + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_mech_ops); +} + +static void +copy_ops_vector_v3(crypto_ops_t *src_ops, crypto_ops_t *dst_ops) +{ + KCF_SPI_COPY_OPS(src_ops, dst_ops, co_nostore_key_ops); +} + +/* + * This routine is used to add cryptographic providers to the KEF framework. + * Providers pass a crypto_provider_info structure to crypto_register_provider() + * and get back a handle. The crypto_provider_info structure contains a + * list of mechanisms supported by the provider and an ops vector containing + * provider entry points. Hardware providers call this routine in their attach + * routines. Software providers call this routine in their _init() routine. + */ +int +crypto_register_provider(crypto_provider_info_t *info, + crypto_kcf_provider_handle_t *handle) +{ + char ks_name[KSTAT_STRLEN]; + + kcf_provider_desc_t *prov_desc = NULL; + int ret = CRYPTO_ARGUMENTS_BAD; + + if (info->pi_interface_version > CRYPTO_SPI_VERSION_3) + return (CRYPTO_VERSION_MISMATCH); + + /* + * Check provider type, must be software, hardware, or logical. + */ + if (info->pi_provider_type != CRYPTO_HW_PROVIDER && + info->pi_provider_type != CRYPTO_SW_PROVIDER && + info->pi_provider_type != CRYPTO_LOGICAL_PROVIDER) + return (CRYPTO_ARGUMENTS_BAD); + + /* + * Allocate and initialize a new provider descriptor. We also + * hold it and release it when done. + */ + prov_desc = kcf_alloc_provider_desc(info); + KCF_PROV_REFHOLD(prov_desc); + + prov_desc->pd_prov_type = info->pi_provider_type; + + /* provider-private handle, opaque to KCF */ + prov_desc->pd_prov_handle = info->pi_provider_handle; + + /* copy provider description string */ + if (info->pi_provider_description != NULL) { + /* + * pi_provider_descriptor is a string that can contain + * up to CRYPTO_PROVIDER_DESCR_MAX_LEN + 1 characters + * INCLUDING the terminating null character. A bcopy() + * is necessary here as pd_description should not have + * a null character. See comments in kcf_alloc_provider_desc() + * for details on pd_description field. + */ + bcopy(info->pi_provider_description, prov_desc->pd_description, + MIN(strlen(info->pi_provider_description), + (size_t)CRYPTO_PROVIDER_DESCR_MAX_LEN)); + } + + if (info->pi_provider_type != CRYPTO_LOGICAL_PROVIDER) { + if (info->pi_ops_vector == NULL) { + goto bail; + } + copy_ops_vector_v1(info->pi_ops_vector, + prov_desc->pd_ops_vector); + if (info->pi_interface_version >= CRYPTO_SPI_VERSION_2) { + copy_ops_vector_v2(info->pi_ops_vector, + prov_desc->pd_ops_vector); + prov_desc->pd_flags = info->pi_flags; + } + if (info->pi_interface_version == CRYPTO_SPI_VERSION_3) { + copy_ops_vector_v3(info->pi_ops_vector, + prov_desc->pd_ops_vector); + } + } + + /* object_ops and nostore_key_ops are mutually exclusive */ + if (prov_desc->pd_ops_vector->co_object_ops && + prov_desc->pd_ops_vector->co_nostore_key_ops) { + goto bail; + } + + /* process the mechanisms supported by the provider */ + if ((ret = init_prov_mechs(info, prov_desc)) != CRYPTO_SUCCESS) + goto bail; + + /* + * Add provider to providers tables, also sets the descriptor + * pd_prov_id field. + */ + if ((ret = kcf_prov_tab_add_provider(prov_desc)) != CRYPTO_SUCCESS) { + undo_register_provider(prov_desc, B_FALSE); + goto bail; + } + + /* + * We create a taskq only for a hardware provider. The global + * software queue is used for software providers. We handle ordering + * of multi-part requests in the taskq routine. So, it is safe to + * have multiple threads for the taskq. We pass TASKQ_PREPOPULATE flag + * to keep some entries cached to improve performance. + */ + if (prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER) + prov_desc->pd_sched_info.ks_taskq = taskq_create("kcf_taskq", + crypto_taskq_threads, minclsyspri, + crypto_taskq_minalloc, crypto_taskq_maxalloc, + TASKQ_PREPOPULATE); + else + prov_desc->pd_sched_info.ks_taskq = NULL; + + /* no kernel session to logical providers */ + if (prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER) { + /* + * Open a session for session-oriented providers. This session + * is used for all kernel consumers. This is fine as a provider + * is required to support multiple thread access to a session. + * We can do this only after the taskq has been created as we + * do a kcf_submit_request() to open the session. + */ + if (KCF_PROV_SESSION_OPS(prov_desc) != NULL) { + kcf_req_params_t params; + + KCF_WRAP_SESSION_OPS_PARAMS(¶ms, + KCF_OP_SESSION_OPEN, &prov_desc->pd_sid, 0, + CRYPTO_USER, NULL, 0, prov_desc); + ret = kcf_submit_request(prov_desc, NULL, NULL, ¶ms, + B_FALSE); + + if (ret != CRYPTO_SUCCESS) { + undo_register_provider(prov_desc, B_TRUE); + ret = CRYPTO_FAILED; + goto bail; + } + } + } + + if (prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER) { + /* + * Create the kstat for this provider. There is a kstat + * installed for each successfully registered provider. + * This kstat is deleted, when the provider unregisters. + */ + if (prov_desc->pd_prov_type == CRYPTO_SW_PROVIDER) { + (void) snprintf(ks_name, KSTAT_STRLEN, "%s_%s", + "NONAME", "provider_stats"); + } else { + (void) snprintf(ks_name, KSTAT_STRLEN, "%s_%d_%u_%s", + "NONAME", 0, + prov_desc->pd_prov_id, "provider_stats"); + } + + prov_desc->pd_kstat = kstat_create("kcf", 0, ks_name, "crypto", + KSTAT_TYPE_NAMED, sizeof (kcf_prov_stats_t) / + sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); + + if (prov_desc->pd_kstat != NULL) { + bcopy(&kcf_stats_ks_data_template, + &prov_desc->pd_ks_data, + sizeof (kcf_stats_ks_data_template)); + prov_desc->pd_kstat->ks_data = &prov_desc->pd_ks_data; + KCF_PROV_REFHOLD(prov_desc); + KCF_PROV_IREFHOLD(prov_desc); + prov_desc->pd_kstat->ks_private = prov_desc; + prov_desc->pd_kstat->ks_update = kcf_prov_kstat_update; + kstat_install(prov_desc->pd_kstat); + } + } + + if (prov_desc->pd_prov_type == CRYPTO_HW_PROVIDER) + process_logical_providers(info, prov_desc); + + mutex_enter(&prov_desc->pd_lock); + prov_desc->pd_state = KCF_PROV_READY; + mutex_exit(&prov_desc->pd_lock); + kcf_do_notify(prov_desc, B_TRUE); + + *handle = prov_desc->pd_kcf_prov_handle; + ret = CRYPTO_SUCCESS; + +bail: + KCF_PROV_REFRELE(prov_desc); + return (ret); +} + +/* + * This routine is used to notify the framework when a provider is being + * removed. Hardware providers call this routine in their detach routines. + * Software providers call this routine in their _fini() routine. + */ +int +crypto_unregister_provider(crypto_kcf_provider_handle_t handle) +{ + uint_t mech_idx; + kcf_provider_desc_t *desc; + kcf_prov_state_t saved_state; + + /* lookup provider descriptor */ + if ((desc = kcf_prov_tab_lookup((crypto_provider_id_t)handle)) == NULL) + return (CRYPTO_UNKNOWN_PROVIDER); + + mutex_enter(&desc->pd_lock); + /* + * Check if any other thread is disabling or removing + * this provider. We return if this is the case. + */ + if (desc->pd_state >= KCF_PROV_DISABLED) { + mutex_exit(&desc->pd_lock); + /* Release reference held by kcf_prov_tab_lookup(). */ + KCF_PROV_REFRELE(desc); + return (CRYPTO_BUSY); + } + + saved_state = desc->pd_state; + desc->pd_state = KCF_PROV_REMOVED; + + if (saved_state == KCF_PROV_BUSY) { + /* + * The per-provider taskq threads may be waiting. We + * signal them so that they can start failing requests. + */ + cv_broadcast(&desc->pd_resume_cv); + } + + if (desc->pd_prov_type == CRYPTO_SW_PROVIDER) { + /* + * Check if this provider is currently being used. + * pd_irefcnt is the number of holds from the internal + * structures. We add one to account for the above lookup. + */ + if (desc->pd_refcnt > desc->pd_irefcnt + 1) { + desc->pd_state = saved_state; + mutex_exit(&desc->pd_lock); + /* Release reference held by kcf_prov_tab_lookup(). */ + KCF_PROV_REFRELE(desc); + /* + * The administrator presumably will stop the clients + * thus removing the holds, when they get the busy + * return value. Any retry will succeed then. + */ + return (CRYPTO_BUSY); + } + } + mutex_exit(&desc->pd_lock); + + if (desc->pd_prov_type != CRYPTO_SW_PROVIDER) { + remove_provider(desc); + } + + if (desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER) { + /* remove the provider from the mechanisms tables */ + for (mech_idx = 0; mech_idx < desc->pd_mech_list_count; + mech_idx++) { + kcf_remove_mech_provider( + desc->pd_mechanisms[mech_idx].cm_mech_name, desc); + } + } + + /* remove provider from providers table */ + if (kcf_prov_tab_rem_provider((crypto_provider_id_t)handle) != + CRYPTO_SUCCESS) { + /* Release reference held by kcf_prov_tab_lookup(). */ + KCF_PROV_REFRELE(desc); + return (CRYPTO_UNKNOWN_PROVIDER); + } + + delete_kstat(desc); + + if (desc->pd_prov_type == CRYPTO_SW_PROVIDER) { + /* Release reference held by kcf_prov_tab_lookup(). */ + KCF_PROV_REFRELE(desc); + + /* + * Wait till the existing requests complete. + */ + mutex_enter(&desc->pd_lock); + while (desc->pd_state != KCF_PROV_FREED) + cv_wait(&desc->pd_remove_cv, &desc->pd_lock); + mutex_exit(&desc->pd_lock); + } else { + /* + * Wait until requests that have been sent to the provider + * complete. + */ + mutex_enter(&desc->pd_lock); + while (desc->pd_irefcnt > 0) + cv_wait(&desc->pd_remove_cv, &desc->pd_lock); + mutex_exit(&desc->pd_lock); + } + + kcf_do_notify(desc, B_FALSE); + + if (desc->pd_prov_type == CRYPTO_SW_PROVIDER) { + /* + * This is the only place where kcf_free_provider_desc() + * is called directly. KCF_PROV_REFRELE() should free the + * structure in all other places. + */ + ASSERT(desc->pd_state == KCF_PROV_FREED && + desc->pd_refcnt == 0); + kcf_free_provider_desc(desc); + } else { + KCF_PROV_REFRELE(desc); + } + + return (CRYPTO_SUCCESS); +} + +/* + * This routine is used to notify the framework that the state of + * a cryptographic provider has changed. Valid state codes are: + * + * CRYPTO_PROVIDER_READY + * The provider indicates that it can process more requests. A provider + * will notify with this event if it previously has notified us with a + * CRYPTO_PROVIDER_BUSY. + * + * CRYPTO_PROVIDER_BUSY + * The provider can not take more requests. + * + * CRYPTO_PROVIDER_FAILED + * The provider encountered an internal error. The framework will not + * be sending any more requests to the provider. The provider may notify + * with a CRYPTO_PROVIDER_READY, if it is able to recover from the error. + * + * This routine can be called from user or interrupt context. + */ +void +crypto_provider_notification(crypto_kcf_provider_handle_t handle, uint_t state) +{ + kcf_provider_desc_t *pd; + + /* lookup the provider from the given handle */ + if ((pd = kcf_prov_tab_lookup((crypto_provider_id_t)handle)) == NULL) + return; + + mutex_enter(&pd->pd_lock); + + if (pd->pd_state <= KCF_PROV_VERIFICATION_FAILED) + goto out; + + if (pd->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + cmn_err(CE_WARN, "crypto_provider_notification: " + "logical provider (%x) ignored\n", handle); + goto out; + } + switch (state) { + case CRYPTO_PROVIDER_READY: + switch (pd->pd_state) { + case KCF_PROV_BUSY: + pd->pd_state = KCF_PROV_READY; + /* + * Signal the per-provider taskq threads that they + * can start submitting requests. + */ + cv_broadcast(&pd->pd_resume_cv); + break; + + case KCF_PROV_FAILED: + /* + * The provider recovered from the error. Let us + * use it now. + */ + pd->pd_state = KCF_PROV_READY; + break; + default: + break; + } + break; + + case CRYPTO_PROVIDER_BUSY: + switch (pd->pd_state) { + case KCF_PROV_READY: + pd->pd_state = KCF_PROV_BUSY; + break; + default: + break; + } + break; + + case CRYPTO_PROVIDER_FAILED: + /* + * We note the failure and return. The per-provider taskq + * threads check this flag and start failing the + * requests, if it is set. See process_req_hwp() for details. + */ + switch (pd->pd_state) { + case KCF_PROV_READY: + pd->pd_state = KCF_PROV_FAILED; + break; + + case KCF_PROV_BUSY: + pd->pd_state = KCF_PROV_FAILED; + /* + * The per-provider taskq threads may be waiting. We + * signal them so that they can start failing requests. + */ + cv_broadcast(&pd->pd_resume_cv); + break; + default: + break; + } + break; + default: + break; + } +out: + mutex_exit(&pd->pd_lock); + KCF_PROV_REFRELE(pd); +} + +/* + * This routine is used to notify the framework the result of + * an asynchronous request handled by a provider. Valid error + * codes are the same as the CRYPTO_* errors defined in common.h. + * + * This routine can be called from user or interrupt context. + */ +void +crypto_op_notification(crypto_req_handle_t handle, int error) +{ + kcf_call_type_t ctype; + + if (handle == NULL) + return; + + if ((ctype = GET_REQ_TYPE(handle)) == CRYPTO_SYNCH) { + kcf_sreq_node_t *sreq = (kcf_sreq_node_t *)handle; + + if (error != CRYPTO_SUCCESS) + sreq->sn_provider->pd_sched_info.ks_nfails++; + KCF_PROV_IREFRELE(sreq->sn_provider); + kcf_sop_done(sreq, error); + } else { + kcf_areq_node_t *areq = (kcf_areq_node_t *)handle; + + ASSERT(ctype == CRYPTO_ASYNCH); + if (error != CRYPTO_SUCCESS) + areq->an_provider->pd_sched_info.ks_nfails++; + KCF_PROV_IREFRELE(areq->an_provider); + kcf_aop_done(areq, error); + } +} + +/* + * This routine is used by software providers to determine + * whether to use KM_SLEEP or KM_NOSLEEP during memory allocation. + * Note that hardware providers can always use KM_SLEEP. So, + * they do not need to call this routine. + * + * This routine can be called from user or interrupt context. + */ +int +crypto_kmflag(crypto_req_handle_t handle) +{ + return (REQHNDL2_KMFLAG(handle)); +} + +/* + * Process the mechanism info structures specified by the provider + * during registration. A NULL crypto_provider_info_t indicates + * an already initialized provider descriptor. + * + * Mechanisms are not added to the kernel's mechanism table if the + * provider is a logical provider. + * + * Returns CRYPTO_SUCCESS on success, CRYPTO_ARGUMENTS if one + * of the specified mechanisms was malformed, or CRYPTO_HOST_MEMORY + * if the table of mechanisms is full. + */ +static int +init_prov_mechs(crypto_provider_info_t *info, kcf_provider_desc_t *desc) +{ + uint_t mech_idx; + uint_t cleanup_idx; + int err = CRYPTO_SUCCESS; + kcf_prov_mech_desc_t *pmd; + int desc_use_count = 0; + int mcount = desc->pd_mech_list_count; + + if (desc->pd_prov_type == CRYPTO_LOGICAL_PROVIDER) { + if (info != NULL) { + ASSERT(info->pi_mechanisms != NULL); + bcopy(info->pi_mechanisms, desc->pd_mechanisms, + sizeof (crypto_mech_info_t) * mcount); + } + return (CRYPTO_SUCCESS); + } + + /* + * Copy the mechanism list from the provider info to the provider + * descriptor. desc->pd_mechanisms has an extra crypto_mech_info_t + * element if the provider has random_ops since we keep an internal + * mechanism, SUN_RANDOM, in this case. + */ + if (info != NULL) { + if (info->pi_ops_vector->co_random_ops != NULL) { + crypto_mech_info_t *rand_mi; + + /* + * Need the following check as it is possible to have + * a provider that implements just random_ops and has + * pi_mechanisms == NULL. + */ + if (info->pi_mechanisms != NULL) { + bcopy(info->pi_mechanisms, desc->pd_mechanisms, + sizeof (crypto_mech_info_t) * (mcount - 1)); + } + rand_mi = &desc->pd_mechanisms[mcount - 1]; + + bzero(rand_mi, sizeof (crypto_mech_info_t)); + (void) strncpy(rand_mi->cm_mech_name, SUN_RANDOM, + CRYPTO_MAX_MECH_NAME); + rand_mi->cm_func_group_mask = CRYPTO_FG_RANDOM; + } else { + ASSERT(info->pi_mechanisms != NULL); + bcopy(info->pi_mechanisms, desc->pd_mechanisms, + sizeof (crypto_mech_info_t) * mcount); + } + } + + /* + * For each mechanism support by the provider, add the provider + * to the corresponding KCF mechanism mech_entry chain. + */ + for (mech_idx = 0; mech_idx < desc->pd_mech_list_count; mech_idx++) { + crypto_mech_info_t *mi = &desc->pd_mechanisms[mech_idx]; + + if ((mi->cm_mech_flags & CRYPTO_KEYSIZE_UNIT_IN_BITS) && + (mi->cm_mech_flags & CRYPTO_KEYSIZE_UNIT_IN_BYTES)) { + err = CRYPTO_ARGUMENTS_BAD; + break; + } + + if (desc->pd_flags & CRYPTO_HASH_NO_UPDATE && + mi->cm_func_group_mask & CRYPTO_FG_DIGEST) { + /* + * We ask the provider to specify the limit + * per hash mechanism. But, in practice, a + * hardware limitation means all hash mechanisms + * will have the same maximum size allowed for + * input data. So, we make it a per provider + * limit to keep it simple. + */ + if (mi->cm_max_input_length == 0) { + err = CRYPTO_ARGUMENTS_BAD; + break; + } else { + desc->pd_hash_limit = mi->cm_max_input_length; + } + } + + if ((err = kcf_add_mech_provider(mech_idx, desc, &pmd)) != + KCF_SUCCESS) + break; + + if (pmd == NULL) + continue; + + /* The provider will be used for this mechanism */ + desc_use_count++; + } + + /* + * Don't allow multiple software providers with disabled mechanisms + * to register. Subsequent enabling of mechanisms will result in + * an unsupported configuration, i.e. multiple software providers + * per mechanism. + */ + if (desc_use_count == 0 && desc->pd_prov_type == CRYPTO_SW_PROVIDER) + return (CRYPTO_ARGUMENTS_BAD); + + if (err == KCF_SUCCESS) + return (CRYPTO_SUCCESS); + + /* + * An error occurred while adding the mechanism, cleanup + * and bail. + */ + for (cleanup_idx = 0; cleanup_idx < mech_idx; cleanup_idx++) { + kcf_remove_mech_provider( + desc->pd_mechanisms[cleanup_idx].cm_mech_name, desc); + } + + if (err == KCF_MECH_TAB_FULL) + return (CRYPTO_HOST_MEMORY); + + return (CRYPTO_ARGUMENTS_BAD); +} + +/* + * Update routine for kstat. Only privileged users are allowed to + * access this information, since this information is sensitive. + * There are some cryptographic attacks (e.g. traffic analysis) + * which can use this information. + */ +static int +kcf_prov_kstat_update(kstat_t *ksp, int rw) +{ + kcf_prov_stats_t *ks_data; + kcf_provider_desc_t *pd = (kcf_provider_desc_t *)ksp->ks_private; + + if (rw == KSTAT_WRITE) + return (EACCES); + + ks_data = ksp->ks_data; + + ks_data->ps_ops_total.value.ui64 = + pd->pd_sched_info.ks_ndispatches; + ks_data->ps_ops_failed.value.ui64 = + pd->pd_sched_info.ks_nfails; + ks_data->ps_ops_busy_rval.value.ui64 = + pd->pd_sched_info.ks_nbusy_rval; + ks_data->ps_ops_passed.value.ui64 = + pd->pd_sched_info.ks_ndispatches - + pd->pd_sched_info.ks_nfails - + pd->pd_sched_info.ks_nbusy_rval; + + return (0); +} + + +/* + * Utility routine called from failure paths in crypto_register_provider() + * and from crypto_load_soft_disabled(). + */ +void +undo_register_provider(kcf_provider_desc_t *desc, boolean_t remove_prov) +{ + uint_t mech_idx; + + /* remove the provider from the mechanisms tables */ + for (mech_idx = 0; mech_idx < desc->pd_mech_list_count; + mech_idx++) { + kcf_remove_mech_provider( + desc->pd_mechanisms[mech_idx].cm_mech_name, desc); + } + + /* remove provider from providers table */ + if (remove_prov) + (void) kcf_prov_tab_rem_provider(desc->pd_prov_id); +} + +/* + * Utility routine called from crypto_load_soft_disabled(). Callers + * should have done a prior undo_register_provider(). + */ +void +redo_register_provider(kcf_provider_desc_t *pd) +{ + /* process the mechanisms supported by the provider */ + (void) init_prov_mechs(NULL, pd); + + /* + * Hold provider in providers table. We should not call + * kcf_prov_tab_add_provider() here as the provider descriptor + * is still valid which means it has an entry in the provider + * table. + */ + KCF_PROV_REFHOLD(pd); + KCF_PROV_IREFHOLD(pd); +} + +/* + * Add provider (p1) to another provider's array of providers (p2). + * Hardware and logical providers use this array to cross-reference + * each other. + */ +static void +add_provider_to_array(kcf_provider_desc_t *p1, kcf_provider_desc_t *p2) +{ + kcf_provider_list_t *new; + + new = kmem_alloc(sizeof (kcf_provider_list_t), KM_SLEEP); + mutex_enter(&p2->pd_lock); + new->pl_next = p2->pd_provider_list; + p2->pd_provider_list = new; + KCF_PROV_IREFHOLD(p1); + new->pl_provider = p1; + mutex_exit(&p2->pd_lock); +} + +/* + * Remove provider (p1) from another provider's array of providers (p2). + * Hardware and logical providers use this array to cross-reference + * each other. + */ +static void +remove_provider_from_array(kcf_provider_desc_t *p1, kcf_provider_desc_t *p2) +{ + + kcf_provider_list_t *pl = NULL, **prev; + + mutex_enter(&p2->pd_lock); + for (pl = p2->pd_provider_list, prev = &p2->pd_provider_list; + pl != NULL; prev = &pl->pl_next, pl = pl->pl_next) { + if (pl->pl_provider == p1) { + break; + } + } + + if (p1 == NULL) { + mutex_exit(&p2->pd_lock); + return; + } + + /* detach and free kcf_provider_list structure */ + KCF_PROV_IREFRELE(p1); + *prev = pl->pl_next; + kmem_free(pl, sizeof (*pl)); + mutex_exit(&p2->pd_lock); +} + +/* + * Convert an array of logical provider handles (crypto_provider_id) + * stored in a crypto_provider_info structure into an array of provider + * descriptors (kcf_provider_desc_t) attached to a logical provider. + */ +static void +process_logical_providers(crypto_provider_info_t *info, kcf_provider_desc_t *hp) +{ + kcf_provider_desc_t *lp; + crypto_provider_id_t handle; + int count = info->pi_logical_provider_count; + int i; + + /* add hardware provider to each logical provider */ + for (i = 0; i < count; i++) { + handle = info->pi_logical_providers[i]; + lp = kcf_prov_tab_lookup((crypto_provider_id_t)handle); + if (lp == NULL) { + continue; + } + add_provider_to_array(hp, lp); + hp->pd_flags |= KCF_LPROV_MEMBER; + + /* + * A hardware provider has to have the provider descriptor of + * every logical provider it belongs to, so it can be removed + * from the logical provider if the hardware provider + * unregisters from the framework. + */ + add_provider_to_array(lp, hp); + KCF_PROV_REFRELE(lp); + } +} + +/* + * This routine removes a provider from all of the logical or + * hardware providers it belongs to, and frees the provider's + * array of pointers to providers. + */ +static void +remove_provider(kcf_provider_desc_t *pp) +{ + kcf_provider_desc_t *p; + kcf_provider_list_t *e, *next; + + mutex_enter(&pp->pd_lock); + for (e = pp->pd_provider_list; e != NULL; e = next) { + p = e->pl_provider; + remove_provider_from_array(pp, p); + if (p->pd_prov_type == CRYPTO_HW_PROVIDER && + p->pd_provider_list == NULL) + p->pd_flags &= ~KCF_LPROV_MEMBER; + KCF_PROV_IREFRELE(p); + next = e->pl_next; + kmem_free(e, sizeof (*e)); + } + pp->pd_provider_list = NULL; + mutex_exit(&pp->pd_lock); +} + +/* + * Dispatch events as needed for a provider. is_added flag tells + * whether the provider is registering or unregistering. + */ +void +kcf_do_notify(kcf_provider_desc_t *prov_desc, boolean_t is_added) +{ + int i; + crypto_notify_event_change_t ec; + + ASSERT(prov_desc->pd_state > KCF_PROV_VERIFICATION_FAILED); + + /* + * Inform interested clients of the mechanisms becoming + * available/unavailable. We skip this for logical providers + * as they do not affect mechanisms. + */ + if (prov_desc->pd_prov_type != CRYPTO_LOGICAL_PROVIDER) { + ec.ec_provider_type = prov_desc->pd_prov_type; + ec.ec_change = is_added ? CRYPTO_MECH_ADDED : + CRYPTO_MECH_REMOVED; + for (i = 0; i < prov_desc->pd_mech_list_count; i++) { + (void) strncpy(ec.ec_mech_name, + prov_desc->pd_mechanisms[i].cm_mech_name, + CRYPTO_MAX_MECH_NAME); + kcf_walk_ntfylist(CRYPTO_EVENT_MECHS_CHANGED, &ec); + } + + } + + /* + * Inform interested clients about the new or departing provider. + * In case of a logical provider, we need to notify the event only + * for the logical provider and not for the underlying + * providers which are known by the KCF_LPROV_MEMBER bit. + */ + if (prov_desc->pd_prov_type == CRYPTO_LOGICAL_PROVIDER || + (prov_desc->pd_flags & KCF_LPROV_MEMBER) == 0) { + kcf_walk_ntfylist(is_added ? CRYPTO_EVENT_PROVIDER_REGISTERED : + CRYPTO_EVENT_PROVIDER_UNREGISTERED, prov_desc); + } +} + +static void +delete_kstat(kcf_provider_desc_t *desc) +{ + /* destroy the kstat created for this provider */ + if (desc->pd_kstat != NULL) { + kcf_provider_desc_t *kspd = desc->pd_kstat->ks_private; + + /* release reference held by desc->pd_kstat->ks_private */ + ASSERT(desc == kspd); + kstat_delete(kspd->pd_kstat); + desc->pd_kstat = NULL; + KCF_PROV_REFRELE(kspd); + KCF_PROV_IREFRELE(kspd); + } +} diff --git a/module/zcommon/Makefile.in b/module/zcommon/Makefile.in index 67e474ee089d..958835edf24b 100644 --- a/module/zcommon/Makefile.in +++ b/module/zcommon/Makefile.in @@ -15,3 +15,6 @@ $(MODULE)-objs += zfs_comutil.o $(MODULE)-objs += zfs_fletcher.o $(MODULE)-objs += zfs_uio.o $(MODULE)-objs += zpool_prop.o + +$(MODULE)-$(CONFIG_X86) += zfs_fletcher_intel.o +$(MODULE)-$(CONFIG_X86) += zfs_fletcher_sse.o diff --git a/module/zcommon/zfs_fletcher.c b/module/zcommon/zfs_fletcher.c index edd0cbe6c611..a3888a32b0f6 100644 --- a/module/zcommon/zfs_fletcher.c +++ b/module/zcommon/zfs_fletcher.c @@ -128,8 +128,78 @@ #include #include #include -#include #include +#include +#include + +static void fletcher_4_scalar_init(zio_cksum_t *zcp); +static void fletcher_4_scalar(const void *buf, uint64_t size, + zio_cksum_t *zcp); +static void fletcher_4_scalar_byteswap(const void *buf, uint64_t size, + zio_cksum_t *zcp); +static boolean_t fletcher_4_scalar_valid(void); + +static const fletcher_4_ops_t fletcher_4_scalar_ops = { + .init = fletcher_4_scalar_init, + .compute = fletcher_4_scalar, + .compute_byteswap = fletcher_4_scalar_byteswap, + .valid = fletcher_4_scalar_valid, + .name = "scalar" +}; + +static const fletcher_4_ops_t *fletcher_4_algos[] = { + &fletcher_4_scalar_ops, +#if defined(HAVE_SSE2) + &fletcher_4_sse2_ops, +#endif +#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) + &fletcher_4_ssse3_ops, +#endif +#if defined(HAVE_AVX) && defined(HAVE_AVX2) + &fletcher_4_avx2_ops, +#endif +}; + +static enum fletcher_selector { + FLETCHER_FASTEST = 0, + FLETCHER_SCALAR, +#if defined(HAVE_SSE2) + FLETCHER_SSE2, +#endif +#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) + FLETCHER_SSSE3, +#endif +#if defined(HAVE_AVX) && defined(HAVE_AVX2) + FLETCHER_AVX2, +#endif + FLETCHER_CYCLE +} fletcher_4_impl_chosen = FLETCHER_SCALAR; + +static struct fletcher_4_impl_selector { + const char *fis_name; + const fletcher_4_ops_t *fis_ops; +} fletcher_4_impl_selectors[] = { + [ FLETCHER_FASTEST ] = { "fastest", NULL }, + [ FLETCHER_SCALAR ] = { "scalar", &fletcher_4_scalar_ops }, +#if defined(HAVE_SSE2) + [ FLETCHER_SSE2 ] = { "sse2", &fletcher_4_sse2_ops }, +#endif +#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) + [ FLETCHER_SSSE3 ] = { "ssse3", &fletcher_4_ssse3_ops }, +#endif +#if defined(HAVE_AVX) && defined(HAVE_AVX2) + [ FLETCHER_AVX2 ] = { "avx2", &fletcher_4_avx2_ops }, +#endif +#if !defined(_KERNEL) + [ FLETCHER_CYCLE ] = { "cycle", &fletcher_4_scalar_ops } +#endif +}; + +static kmutex_t fletcher_4_impl_lock; + +static kstat_t *fletcher_4_kstat; + +static kstat_named_t fletcher_4_kstat_data[ARRAY_SIZE(fletcher_4_algos)]; void fletcher_2_native(const void *buf, uint64_t size, zio_cksum_t *zcp) @@ -165,32 +235,25 @@ fletcher_2_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp) ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1); } -void -fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp) +static void fletcher_4_scalar_init(zio_cksum_t *zcp) { - const uint32_t *ip = buf; - const uint32_t *ipend = ip + (size / sizeof (uint32_t)); - uint64_t a, b, c, d; - - for (a = b = c = d = 0; ip < ipend; ip++) { - a += ip[0]; - b += a; - c += b; - d += c; - } - - ZIO_SET_CHECKSUM(zcp, a, b, c, d); + ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); } -void -fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp) +static void +fletcher_4_scalar(const void *buf, uint64_t size, zio_cksum_t *zcp) { const uint32_t *ip = buf; const uint32_t *ipend = ip + (size / sizeof (uint32_t)); uint64_t a, b, c, d; - for (a = b = c = d = 0; ip < ipend; ip++) { - a += BSWAP_32(ip[0]); + a = zcp->zc_word[0]; + b = zcp->zc_word[1]; + c = zcp->zc_word[2]; + d = zcp->zc_word[3]; + + for (; ip < ipend; ip++) { + a += ip[0]; b += a; c += b; d += c; @@ -199,9 +262,8 @@ fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp) ZIO_SET_CHECKSUM(zcp, a, b, c, d); } -void -fletcher_4_incremental_native(const void *buf, uint64_t size, - zio_cksum_t *zcp) +static void +fletcher_4_scalar_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp) { const uint32_t *ip = buf; const uint32_t *ipend = ip + (size / sizeof (uint32_t)); @@ -213,7 +275,7 @@ fletcher_4_incremental_native(const void *buf, uint64_t size, d = zcp->zc_word[3]; for (; ip < ipend; ip++) { - a += ip[0]; + a += BSWAP_32(ip[0]); b += a; c += b; d += c; @@ -222,30 +284,236 @@ fletcher_4_incremental_native(const void *buf, uint64_t size, ZIO_SET_CHECKSUM(zcp, a, b, c, d); } +static boolean_t +fletcher_4_scalar_valid(void) +{ + return (B_TRUE); +} + +int +fletcher_4_impl_set(const char *val) +{ + const fletcher_4_ops_t *ops; + enum fletcher_selector idx; + size_t val_len; + unsigned i; + + val_len = strlen(val); + while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */ + val_len--; + + for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) { + const char *name = fletcher_4_impl_selectors[i].fis_name; + + if (val_len == strlen(name) && + strncmp(val, name, val_len) == 0) { + idx = i; + break; + } + } + if (i >= ARRAY_SIZE(fletcher_4_impl_selectors)) + return (-EINVAL); + + ops = fletcher_4_impl_selectors[idx].fis_ops; + if (ops == NULL || !ops->valid()) + return (-ENOTSUP); + + mutex_enter(&fletcher_4_impl_lock); + if (fletcher_4_impl_chosen != idx) + fletcher_4_impl_chosen = idx; + mutex_exit(&fletcher_4_impl_lock); + + return (0); +} + +static inline const fletcher_4_ops_t * +fletcher_4_impl_get(void) +{ +#if !defined(_KERNEL) + if (fletcher_4_impl_chosen == FLETCHER_CYCLE) { + static volatile unsigned int cycle_count = 0; + const fletcher_4_ops_t *ops = NULL; + unsigned int index; + + while (1) { + index = atomic_inc_uint_nv(&cycle_count); + ops = fletcher_4_algos[ + index % ARRAY_SIZE(fletcher_4_algos)]; + if (ops->valid()) + break; + } + return (ops); + } +#endif + membar_producer(); + return (fletcher_4_impl_selectors[fletcher_4_impl_chosen].fis_ops); +} + +void +fletcher_4_native(const void *buf, uint64_t size, zio_cksum_t *zcp) +{ + const fletcher_4_ops_t *ops; + + if (IS_P2ALIGNED(size, 4 * sizeof (uint32_t))) + ops = fletcher_4_impl_get(); + else + ops = &fletcher_4_scalar_ops; + + ops->init(zcp); + ops->compute(buf, size, zcp); + if (ops->fini != NULL) + ops->fini(zcp); +} + +void +fletcher_4_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp) +{ + const fletcher_4_ops_t *ops; + + if (IS_P2ALIGNED(size, 4 * sizeof (uint32_t))) + ops = fletcher_4_impl_get(); + else + ops = &fletcher_4_scalar_ops; + + ops->init(zcp); + ops->compute_byteswap(buf, size, zcp); + if (ops->fini != NULL) + ops->fini(zcp); +} + +void +fletcher_4_incremental_native(const void *buf, uint64_t size, + zio_cksum_t *zcp) +{ + fletcher_4_scalar(buf, size, zcp); +} + void fletcher_4_incremental_byteswap(const void *buf, uint64_t size, zio_cksum_t *zcp) { - const uint32_t *ip = buf; - const uint32_t *ipend = ip + (size / sizeof (uint32_t)); - uint64_t a, b, c, d; + fletcher_4_scalar_byteswap(buf, size, zcp); +} - a = zcp->zc_word[0]; - b = zcp->zc_word[1]; - c = zcp->zc_word[2]; - d = zcp->zc_word[3]; +void +fletcher_4_init(void) +{ + const uint64_t const bench_ns = (50 * MICROSEC); /* 50ms */ + unsigned long best_run_count = 0; + unsigned long best_run_index = 0; + const unsigned data_size = 4096; + char *databuf; + int i; - for (; ip < ipend; ip++) { - a += BSWAP_32(ip[0]); - b += a; - c += b; - d += c; + databuf = kmem_alloc(data_size, KM_SLEEP); + for (i = 0; i < ARRAY_SIZE(fletcher_4_algos); i++) { + const fletcher_4_ops_t *ops = fletcher_4_algos[i]; + kstat_named_t *stat = &fletcher_4_kstat_data[i]; + unsigned long run_count = 0; + hrtime_t start; + zio_cksum_t zc; + + strncpy(stat->name, ops->name, sizeof (stat->name) - 1); + stat->data_type = KSTAT_DATA_UINT64; + stat->value.ui64 = 0; + + if (!ops->valid()) + continue; + + kpreempt_disable(); + start = gethrtime(); + ops->init(&zc); + do { + ops->compute(databuf, data_size, &zc); + ops->compute_byteswap(databuf, data_size, &zc); + run_count++; + } while (gethrtime() < start + bench_ns); + if (ops->fini != NULL) + ops->fini(&zc); + kpreempt_enable(); + + if (run_count > best_run_count) { + best_run_count = run_count; + best_run_index = i; + } + + /* + * Due to high overhead of gethrtime(), the performance data + * here is inaccurate and much slower than it could be. + * It's fine for our use though because only relative speed + * is important. + */ + stat->value.ui64 = data_size * run_count * + (NANOSEC / bench_ns) >> 20; /* by MB/s */ } + kmem_free(databuf, data_size); - ZIO_SET_CHECKSUM(zcp, a, b, c, d); + fletcher_4_impl_selectors[FLETCHER_FASTEST].fis_ops = + fletcher_4_algos[best_run_index]; + + mutex_init(&fletcher_4_impl_lock, NULL, MUTEX_DEFAULT, NULL); + fletcher_4_impl_set("fastest"); + + fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench", + "misc", KSTAT_TYPE_NAMED, ARRAY_SIZE(fletcher_4_algos), + KSTAT_FLAG_VIRTUAL); + if (fletcher_4_kstat != NULL) { + fletcher_4_kstat->ks_data = fletcher_4_kstat_data; + kstat_install(fletcher_4_kstat); + } +} + +void +fletcher_4_fini(void) +{ + mutex_destroy(&fletcher_4_impl_lock); + if (fletcher_4_kstat != NULL) { + kstat_delete(fletcher_4_kstat); + fletcher_4_kstat = NULL; + } } #if defined(_KERNEL) && defined(HAVE_SPL) + +static int +fletcher_4_param_get(char *buffer, struct kernel_param *unused) +{ + int i, cnt = 0; + + for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) { + const fletcher_4_ops_t *ops; + + ops = fletcher_4_impl_selectors[i].fis_ops; + if (!ops->valid()) + continue; + + cnt += sprintf(buffer + cnt, + fletcher_4_impl_chosen == i ? "[%s] " : "%s ", + fletcher_4_impl_selectors[i].fis_name); + } + + return (cnt); +} + +static int +fletcher_4_param_set(const char *val, struct kernel_param *unused) +{ + return (fletcher_4_impl_set(val)); +} + +/* + * Choose a fletcher 4 implementation in ZFS. + * Users can choose the "fastest" algorithm, or "scalar" and "avx2" which means + * to compute fletcher 4 by CPU or vector instructions respectively. + * Users can also choose "cycle" to exercise all implementions, but this is + * for testing purpose therefore it can only be set in user space. + */ +module_param_call(zfs_fletcher_4_impl, + fletcher_4_param_set, fletcher_4_param_get, NULL, 0644); +MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 algorithm"); + +EXPORT_SYMBOL(fletcher_4_init); +EXPORT_SYMBOL(fletcher_4_fini); EXPORT_SYMBOL(fletcher_2_native); EXPORT_SYMBOL(fletcher_2_byteswap); EXPORT_SYMBOL(fletcher_4_native); diff --git a/module/zcommon/zfs_fletcher_intel.c b/module/zcommon/zfs_fletcher_intel.c new file mode 100644 index 000000000000..38a40e234c43 --- /dev/null +++ b/module/zcommon/zfs_fletcher_intel.c @@ -0,0 +1,148 @@ +/* + * Implement fast Fletcher4 with AVX2 instructions. (x86_64) + * + * Use the 256-bit AVX2 SIMD instructions and registers to compute + * Fletcher4 in four incremental 64-bit parallel accumulator streams, + * and then combine the streams to form the final four checksum words. + * + * Copyright (C) 2015 Intel Corporation. + * + * Authors: + * James Guilford + * Jinshan Xiong + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(HAVE_AVX) && defined(HAVE_AVX2) + +#include +#include +#include + +static void +fletcher_4_avx2_init(zio_cksum_t *zcp) +{ + kfpu_begin(); + + /* clear avx2 registers */ + asm volatile("vpxor %ymm0, %ymm0, %ymm0"); + asm volatile("vpxor %ymm1, %ymm1, %ymm1"); + asm volatile("vpxor %ymm2, %ymm2, %ymm2"); + asm volatile("vpxor %ymm3, %ymm3, %ymm3"); +} + +static void +fletcher_4_avx2_fini(zio_cksum_t *zcp) +{ + uint64_t __attribute__((aligned(32))) a[4]; + uint64_t __attribute__((aligned(32))) b[4]; + uint64_t __attribute__((aligned(32))) c[4]; + uint64_t __attribute__((aligned(32))) d[4]; + uint64_t A, B, C, D; + + asm volatile("vmovdqu %%ymm0, %0":"=m" (a)); + asm volatile("vmovdqu %%ymm1, %0":"=m" (b)); + asm volatile("vmovdqu %%ymm2, %0":"=m" (c)); + asm volatile("vmovdqu %%ymm3, %0":"=m" (d)); + asm volatile("vzeroupper"); + + kfpu_end(); + + A = a[0] + a[1] + a[2] + a[3]; + B = 0 - a[1] - 2*a[2] - 3*a[3] + + 4*b[0] + 4*b[1] + 4*b[2] + 4*b[3]; + + C = a[2] + 3*a[3] + - 6*b[0] - 10*b[1] - 14*b[2] - 18*b[3] + + 16*c[0] + 16*c[1] + 16*c[2] + 16*c[3]; + + D = 0 - a[3] + + 4*b[0] + 10*b[1] + 20*b[2] + 34*b[3] + - 48*c[0] - 64*c[1] - 80*c[2] - 96*c[3] + + 64*d[0] + 64*d[1] + 64*d[2] + 64*d[3]; + + ZIO_SET_CHECKSUM(zcp, A, B, C, D); +} + +static void +fletcher_4_avx2(const void *buf, uint64_t size, zio_cksum_t *unused) +{ + const uint64_t *ip = buf; + const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); + + for (; ip < ipend; ip += 2) { + asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip)); + asm volatile("vpaddq %ymm4, %ymm0, %ymm0"); + asm volatile("vpaddq %ymm0, %ymm1, %ymm1"); + asm volatile("vpaddq %ymm1, %ymm2, %ymm2"); + asm volatile("vpaddq %ymm2, %ymm3, %ymm3"); + } +} + +static void +fletcher_4_avx2_byteswap(const void *buf, uint64_t size, zio_cksum_t *unused) +{ + static const struct { + uint64_t v[4] __attribute__((aligned(32))); + } mask = { + .v = { 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B, + 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B } + }; + const uint64_t *ip = buf; + const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); + + asm volatile("vmovdqa %0, %%ymm5"::"m"(mask)); + + for (; ip < ipend; ip += 2) { + asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip)); + asm volatile("vpshufb %ymm5, %ymm4, %ymm4"); + + asm volatile("vpaddq %ymm4, %ymm0, %ymm0"); + asm volatile("vpaddq %ymm0, %ymm1, %ymm1"); + asm volatile("vpaddq %ymm1, %ymm2, %ymm2"); + asm volatile("vpaddq %ymm2, %ymm3, %ymm3"); + } +} + +static boolean_t fletcher_4_avx2_valid(void) +{ + return (zfs_avx_available() && zfs_avx2_available()); +} + +const fletcher_4_ops_t fletcher_4_avx2_ops = { + .init = fletcher_4_avx2_init, + .fini = fletcher_4_avx2_fini, + .compute = fletcher_4_avx2, + .compute_byteswap = fletcher_4_avx2_byteswap, + .valid = fletcher_4_avx2_valid, + .name = "avx2" +}; + +#endif /* defined(HAVE_AVX) && defined(HAVE_AVX2) */ diff --git a/module/zcommon/zfs_fletcher_sse.c b/module/zcommon/zfs_fletcher_sse.c new file mode 100644 index 000000000000..2a4e6a3f2288 --- /dev/null +++ b/module/zcommon/zfs_fletcher_sse.c @@ -0,0 +1,205 @@ +/* + * Implement fast Fletcher4 with SSE2,SSSE3 instructions. (x86) + * + * Use the 128-bit SSE2/SSSE3 SIMD instructions and registers to compute + * Fletcher4 in four incremental 64-bit parallel accumulator streams, + * and then combine the streams to form the final four checksum words. + * This implementation is a derivative of the AVX SIMD implementation by + * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c). + * + * Copyright (C) 2016 Tyler J. Stachecki. + * + * Authors: + * Tyler J. Stachecki + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(HAVE_SSE2) + +#include +#include +#include + +struct zfs_fletcher_sse_array { + uint64_t v[2] __attribute__((aligned(16))); +}; + +static void +fletcher_4_sse2_init(zio_cksum_t *zcp) +{ + kfpu_begin(); + + /* clear sse registers */ + asm volatile("pxor %xmm0, %xmm0"); + asm volatile("pxor %xmm1, %xmm1"); + asm volatile("pxor %xmm2, %xmm2"); + asm volatile("pxor %xmm3, %xmm3"); +} + +static void +fletcher_4_sse2_fini(zio_cksum_t *zcp) +{ + struct zfs_fletcher_sse_array a, b, c, d; + uint64_t A, B, C, D; + + asm volatile("movdqu %%xmm0, %0":"=m" (a.v)); + asm volatile("movdqu %%xmm1, %0":"=m" (b.v)); + asm volatile("psllq $0x2, %xmm2"); + asm volatile("movdqu %%xmm2, %0":"=m" (c.v)); + asm volatile("psllq $0x3, %xmm3"); + asm volatile("movdqu %%xmm3, %0":"=m" (d.v)); + + kfpu_end(); + + /* + * The mixing matrix for checksum calculation is: + * a = a0 + a1 + * b = 2b0 + 2b1 - a1 + * c = 4c0 - b0 + 4c1 -3b1 + * d = 8d0 - 4c0 + 8d1 - 8c1 + b1; + * + * c and d are multiplied by 4 and 8, respectively, + * before spilling the vectors out to memory. + */ + A = a.v[0] + a.v[1]; + B = 2*b.v[0] + 2*b.v[1] - a.v[1]; + C = c.v[0] - b.v[0] + c.v[1] - 3*b.v[1]; + D = d.v[0] - c.v[0] + d.v[1] - 2*c.v[1] + b.v[1]; + + ZIO_SET_CHECKSUM(zcp, A, B, C, D); +} + +static void +fletcher_4_sse2(const void *buf, uint64_t size, zio_cksum_t *unused) +{ + const uint64_t *ip = buf; + const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); + + asm volatile("pxor %xmm4, %xmm4"); + + for (; ip < ipend; ip += 2) { + asm volatile("movdqu %0, %%xmm5" :: "m"(*ip)); + asm volatile("movdqa %xmm5, %xmm6"); + asm volatile("punpckldq %xmm4, %xmm5"); + asm volatile("punpckhdq %xmm4, %xmm6"); + asm volatile("paddq %xmm5, %xmm0"); + asm volatile("paddq %xmm0, %xmm1"); + asm volatile("paddq %xmm1, %xmm2"); + asm volatile("paddq %xmm2, %xmm3"); + asm volatile("paddq %xmm6, %xmm0"); + asm volatile("paddq %xmm0, %xmm1"); + asm volatile("paddq %xmm1, %xmm2"); + asm volatile("paddq %xmm2, %xmm3"); + } +} + +static void +fletcher_4_sse2_byteswap(const void *buf, uint64_t size, zio_cksum_t *unused) +{ + const uint32_t *ip = buf; + const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); + + for (; ip < ipend; ip += 2) { + uint32_t scratch; + + asm volatile("bswapl %0" : "=r"(scratch) : "0"(*ip)); + asm volatile("movd %0, %%xmm5" :: "r"(scratch)); + asm volatile("bswapl %0" : "=r"(scratch) : "0"(*(ip + 1))); + asm volatile("movd %0, %%xmm6" :: "r"(scratch)); + asm volatile("punpcklqdq %xmm6, %xmm5"); + asm volatile("paddq %xmm5, %xmm0"); + asm volatile("paddq %xmm0, %xmm1"); + asm volatile("paddq %xmm1, %xmm2"); + asm volatile("paddq %xmm2, %xmm3"); + } +} + +static boolean_t fletcher_4_sse2_valid(void) +{ + return (zfs_sse2_available()); +} + +const fletcher_4_ops_t fletcher_4_sse2_ops = { + .init = fletcher_4_sse2_init, + .fini = fletcher_4_sse2_fini, + .compute = fletcher_4_sse2, + .compute_byteswap = fletcher_4_sse2_byteswap, + .valid = fletcher_4_sse2_valid, + .name = "sse2" +}; + +#endif /* defined(HAVE_SSE2) */ + +#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) +static void +fletcher_4_ssse3_byteswap(const void *buf, uint64_t size, zio_cksum_t *unused) +{ + static const struct zfs_fletcher_sse_array mask = { + .v = { 0x0405060700010203, 0x0C0D0E0F08090A0B } + }; + + const uint64_t *ip = buf; + const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); + + asm volatile("movdqu %0, %%xmm7"::"m" (mask)); + asm volatile("pxor %xmm4, %xmm4"); + + for (; ip < ipend; ip += 2) { + asm volatile("movdqu %0, %%xmm5"::"m" (*ip)); + asm volatile("pshufb %xmm7, %xmm5"); + asm volatile("movdqa %xmm5, %xmm6"); + asm volatile("punpckldq %xmm4, %xmm5"); + asm volatile("punpckhdq %xmm4, %xmm6"); + asm volatile("paddq %xmm5, %xmm0"); + asm volatile("paddq %xmm0, %xmm1"); + asm volatile("paddq %xmm1, %xmm2"); + asm volatile("paddq %xmm2, %xmm3"); + asm volatile("paddq %xmm6, %xmm0"); + asm volatile("paddq %xmm0, %xmm1"); + asm volatile("paddq %xmm1, %xmm2"); + asm volatile("paddq %xmm2, %xmm3"); + } +} + +static boolean_t fletcher_4_ssse3_valid(void) +{ + return (zfs_sse2_available() && zfs_ssse3_available()); +} + +const fletcher_4_ops_t fletcher_4_ssse3_ops = { + .init = fletcher_4_sse2_init, + .fini = fletcher_4_sse2_fini, + .compute = fletcher_4_sse2, + .compute_byteswap = fletcher_4_ssse3_byteswap, + .valid = fletcher_4_ssse3_valid, + .name = "ssse3" +}; + +#endif /* defined(HAVE_SSE2) && defined(HAVE_SSSE3) */ diff --git a/module/zcommon/zfs_namecheck.c b/module/zcommon/zfs_namecheck.c index ff724be588cc..b58071bed055 100644 --- a/module/zcommon/zfs_namecheck.c +++ b/module/zcommon/zfs_namecheck.c @@ -69,7 +69,7 @@ zfs_component_namecheck(const char *path, namecheck_err_t *why, char *what) { const char *loc; - if (strlen(path) >= MAXNAMELEN) { + if (strlen(path) >= ZFS_MAX_DATASET_NAME_LEN) { if (why) *why = NAME_ERR_TOOLONG; return (-1); @@ -140,27 +140,8 @@ dataset_namecheck(const char *path, namecheck_err_t *why, char *what) /* * Make sure the name is not too long. - * - * ZFS_MAXNAMELEN is the maximum dataset length used in the userland - * which is the same as MAXNAMELEN used in the kernel. - * If ZFS_MAXNAMELEN value is changed, make sure to cleanup all - * places using MAXNAMELEN. - * - * When HAVE_KOBJ_NAME_LEN is defined the maximum safe kobject name - * length is 20 bytes. This 20 bytes is broken down as follows to - * provide a maximum safe /[@snapshot] length of only - * 18 bytes. To ensure bytes are left for [@snapshot] the - * portition is futher limited to 9 bytes. For 2.6.27 and - * newer kernels this limit is set to MAXNAMELEN. - * - * / + + - * (18) + (1) + (1) */ -#ifdef HAVE_KOBJ_NAME_LEN - if (strlen(path) > 18) { -#else - if (strlen(path) >= MAXNAMELEN) { -#endif /* HAVE_KOBJ_NAME_LEN */ + if (strlen(path) >= ZFS_MAX_DATASET_NAME_LEN) { if (why) *why = NAME_ERR_TOOLONG; return (-1); @@ -289,7 +270,7 @@ mountpoint_namecheck(const char *path, namecheck_err_t *why) while (*end != '/' && *end != '\0') end++; - if (end - start >= MAXNAMELEN) { + if (end - start >= ZFS_MAX_DATASET_NAME_LEN) { if (why) *why = NAME_ERR_TOOLONG; return (-1); @@ -314,27 +295,8 @@ pool_namecheck(const char *pool, namecheck_err_t *why, char *what) /* * Make sure the name is not too long. - * - * ZPOOL_MAXNAMELEN is the maximum pool length used in the userland - * which is the same as MAXNAMELEN used in the kernel. - * If ZPOOL_MAXNAMELEN value is changed, make sure to cleanup all - * places using MAXNAMELEN. - * - * When HAVE_KOBJ_NAME_LEN is defined the maximum safe kobject name - * length is 20 bytes. This 20 bytes is broken down as follows to - * provide a maximum safe /[@snapshot] length of only - * 18 bytes. To ensure bytes are left for [@snapshot] the - * portition is futher limited to 8 bytes. For 2.6.27 and - * newer kernels this limit is set to MAXNAMELEN. - * - * / + + - * (18) + (1) + (1) */ -#ifdef HAVE_KOBJ_NAME_LEN - if (strlen(pool) > 8) { -#else - if (strlen(pool) >= MAXNAMELEN) { -#endif /* HAVE_KOBJ_NAME_LEN */ + if (strlen(pool) >= ZFS_MAX_DATASET_NAME_LEN) { if (why) *why = NAME_ERR_TOOLONG; return (-1); diff --git a/module/zcommon/zfs_prop.c b/module/zcommon/zfs_prop.c index 5a88cbe6c7b0..1d68ca29e6d6 100644 --- a/module/zcommon/zfs_prop.c +++ b/module/zcommon/zfs_prop.c @@ -35,6 +35,7 @@ #include "zfs_prop.h" #include "zfs_deleg.h" +#include "zfs_fletcher.h" #if defined(_KERNEL) #include @@ -210,6 +211,17 @@ zfs_prop_init(void) { NULL } }; + static zprop_index_t dnsize_table[] = { + { "legacy", ZFS_DNSIZE_LEGACY }, + { "auto", ZFS_DNSIZE_AUTO }, + { "1k", ZFS_DNSIZE_1K }, + { "2k", ZFS_DNSIZE_2K }, + { "4k", ZFS_DNSIZE_4K }, + { "8k", ZFS_DNSIZE_8K }, + { "16k", ZFS_DNSIZE_16K }, + { NULL } + }; + static zprop_index_t redundant_metadata_table[] = { { "all", ZFS_REDUNDANT_METADATA_ALL }, { "most", ZFS_REDUNDANT_METADATA_MOST }, @@ -270,6 +282,9 @@ zfs_prop_init(void) zprop_register_index(ZFS_PROP_XATTR, "xattr", ZFS_XATTR_DIR, PROP_INHERIT, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_SNAPSHOT, "on | off | dir | sa", "XATTR", xattr_table); + zprop_register_index(ZFS_PROP_DNODESIZE, "dnodesize", + ZFS_DNSIZE_LEGACY, PROP_INHERIT, ZFS_TYPE_FILESYSTEM, + "legacy | auto | 1k | 2k | 4k | 8k | 16k", "DNSIZE", dnsize_table); /* inherit index (boolean) properties */ zprop_register_index(ZFS_PROP_ATIME, "atime", 1, PROP_INHERIT, @@ -360,6 +375,10 @@ zfs_prop_init(void) zprop_register_string(ZFS_PROP_SELINUX_ROOTCONTEXT, "rootcontext", "none", PROP_DEFAULT, ZFS_TYPE_DATASET, "", "ROOTCONTEXT"); + zprop_register_string(ZFS_PROP_RECEIVE_RESUME_TOKEN, + "receive_resume_token", + NULL, PROP_READONLY, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, + "", "RESUMETOK"); /* readonly number properties */ zprop_register_number(ZFS_PROP_USED, "used", 0, PROP_READONLY, @@ -695,12 +714,14 @@ zfs_prop_align_right(zfs_prop_t prop) static int __init zcommon_init(void) { + fletcher_4_init(); return (0); } static void __exit zcommon_fini(void) { + fletcher_4_fini(); } module_init(zcommon_init); diff --git a/module/zcommon/zpool_prop.c b/module/zcommon/zpool_prop.c index 910c56dcc2a9..4a5836e5b739 100644 --- a/module/zcommon/zpool_prop.c +++ b/module/zcommon/zpool_prop.c @@ -135,6 +135,8 @@ zpool_prop_init(void) PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_POOL, "MAXBLOCKSIZE"); zprop_register_hidden(ZPOOL_PROP_TNAME, "tname", PROP_TYPE_STRING, PROP_ONETIME, ZFS_TYPE_POOL, "TNAME"); + zprop_register_hidden(ZPOOL_PROP_MAXDNODESIZE, "maxdnodesize", + PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_POOL, "MAXDNODESIZE"); } /* diff --git a/module/zfs/Makefile.in b/module/zfs/Makefile.in index f33faf157cc8..07c6d1c6ca1b 100644 --- a/module/zfs/Makefile.in +++ b/module/zfs/Makefile.in @@ -43,6 +43,7 @@ $(MODULE)-objs += lz4.o $(MODULE)-objs += metaslab.o $(MODULE)-objs += multilist.o $(MODULE)-objs += pathname.o +$(MODULE)-objs += policy.o $(MODULE)-objs += range_tree.o $(MODULE)-objs += refcount.o $(MODULE)-objs += rrwlock.o @@ -70,6 +71,8 @@ $(MODULE)-objs += vdev_mirror.o $(MODULE)-objs += vdev_missing.o $(MODULE)-objs += vdev_queue.o $(MODULE)-objs += vdev_raidz.o +$(MODULE)-objs += vdev_raidz_math.o +$(MODULE)-objs += vdev_raidz_math_scalar.o $(MODULE)-objs += vdev_root.o $(MODULE)-objs += zap.o $(MODULE)-objs += zap_leaf.o @@ -108,3 +111,7 @@ $(MODULE)-objs += zrlock.o $(MODULE)-objs += zvol.o $(MODULE)-objs += dsl_destroy.o $(MODULE)-objs += dsl_userhold.o + +$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_sse2.o +$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_ssse3.o +$(MODULE)-$(CONFIG_X86) += vdev_raidz_math_avx2.o diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 716ba5c2d95d..6d8bd48a3b18 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, Joyent, Inc. All rights reserved. - * Copyright (c) 2011, 2015 by Delphix. All rights reserved. + * Copyright (c) 2011, 2016 by Delphix. All rights reserved. * Copyright (c) 2014 by Saso Kiselkov. All rights reserved. * Copyright 2014 Nexenta Systems, Inc. All rights reserved. */ @@ -231,6 +231,8 @@ unsigned long zfs_arc_max = 0; unsigned long zfs_arc_min = 0; unsigned long zfs_arc_meta_limit = 0; unsigned long zfs_arc_meta_min = 0; +unsigned long zfs_arc_dnode_limit = 0; +unsigned long zfs_arc_dnode_reduce_percent = 10; int zfs_arc_grow_retry = 0; int zfs_arc_shrink_shift = 0; int zfs_arc_p_min_shift = 0; @@ -328,13 +330,17 @@ typedef struct arc_stats { */ kstat_named_t arcstat_metadata_size; /* - * Number of bytes consumed by various buffers and structures - * not actually backed with ARC buffers. This includes bonus - * buffers (allocated directly via zio_buf_* functions), - * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t - * cache), and dnode_t structures (allocated via dnode_t cache). + * Number of bytes consumed by dmu_buf_impl_t objects. */ - kstat_named_t arcstat_other_size; + kstat_named_t arcstat_dbuf_size; + /* + * Number of bytes consumed by dnode_t objects. + */ + kstat_named_t arcstat_dnode_size; + /* + * Number of bytes consumed by bonus buffers. + */ + kstat_named_t arcstat_bonus_size; /* * Total number of bytes consumed by ARC buffers residing in the * arc_anon state. This includes *all* buffers in the arc_anon @@ -473,6 +479,7 @@ typedef struct arc_stats { kstat_named_t arcstat_prune; kstat_named_t arcstat_meta_used; kstat_named_t arcstat_meta_limit; + kstat_named_t arcstat_dnode_limit; kstat_named_t arcstat_meta_max; kstat_named_t arcstat_meta_min; kstat_named_t arcstat_sync_wait_for_async; @@ -517,7 +524,9 @@ static arc_stats_t arc_stats = { { "hdr_size", KSTAT_DATA_UINT64 }, { "data_size", KSTAT_DATA_UINT64 }, { "metadata_size", KSTAT_DATA_UINT64 }, - { "other_size", KSTAT_DATA_UINT64 }, + { "dbuf_size", KSTAT_DATA_UINT64 }, + { "dnode_size", KSTAT_DATA_UINT64 }, + { "bonus_size", KSTAT_DATA_UINT64 }, { "anon_size", KSTAT_DATA_UINT64 }, { "anon_evictable_data", KSTAT_DATA_UINT64 }, { "anon_evictable_metadata", KSTAT_DATA_UINT64 }, @@ -570,6 +579,7 @@ static arc_stats_t arc_stats = { { "arc_prune", KSTAT_DATA_UINT64 }, { "arc_meta_used", KSTAT_DATA_UINT64 }, { "arc_meta_limit", KSTAT_DATA_UINT64 }, + { "arc_dnode_limit", KSTAT_DATA_UINT64 }, { "arc_meta_max", KSTAT_DATA_UINT64 }, { "arc_meta_min", KSTAT_DATA_UINT64 }, { "sync_wait_for_async", KSTAT_DATA_UINT64 }, @@ -641,9 +651,13 @@ static arc_state_t *arc_l2c_only; #define arc_tempreserve ARCSTAT(arcstat_tempreserve) #define arc_loaned_bytes ARCSTAT(arcstat_loaned_bytes) #define arc_meta_limit ARCSTAT(arcstat_meta_limit) /* max size for metadata */ +#define arc_dnode_limit ARCSTAT(arcstat_dnode_limit) /* max size for dnodes */ #define arc_meta_min ARCSTAT(arcstat_meta_min) /* min size for metadata */ #define arc_meta_used ARCSTAT(arcstat_meta_used) /* size of metadata */ #define arc_meta_max ARCSTAT(arcstat_meta_max) /* max size of metadata */ +#define arc_dbuf_size ARCSTAT(arcstat_dbuf_size) /* dbuf metadata */ +#define arc_dnode_size ARCSTAT(arcstat_dnode_size) /* dnode metadata */ +#define arc_bonus_size ARCSTAT(arcstat_bonus_size) /* bonus buffer metadata */ #define arc_need_free ARCSTAT(arcstat_need_free) /* bytes to be freed */ #define arc_sys_free ARCSTAT(arcstat_sys_free) /* target system free bytes */ @@ -803,6 +817,7 @@ static void arc_access(arc_buf_hdr_t *, kmutex_t *); static boolean_t arc_is_overflowing(void); static void arc_buf_watch(arc_buf_t *); static void arc_tuning_update(void); +static void arc_prune_async(int64_t); static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *); static uint32_t arc_bufc_to_flags(arc_buf_contents_t); @@ -1461,6 +1476,13 @@ arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index) l2arc_buf_hdr_t *l2hdr = NULL; arc_state_t *state = NULL; + memset(abi, 0, sizeof (arc_buf_info_t)); + + if (hdr == NULL) + return; + + abi->abi_flags = hdr->b_flags; + if (HDR_HAS_L1HDR(hdr)) { l1hdr = &hdr->b_l1hdr; state = l1hdr->b_state; @@ -1468,9 +1490,6 @@ arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index) if (HDR_HAS_L2HDR(hdr)) l2hdr = &hdr->b_l2hdr; - memset(abi, 0, sizeof (arc_buf_info_t)); - abi->abi_flags = hdr->b_flags; - if (l1hdr) { abi->abi_datacnt = l1hdr->b_datacnt; abi->abi_access = l1hdr->b_arc_access; @@ -1676,8 +1695,14 @@ arc_space_consume(uint64_t space, arc_space_type_t type) case ARC_SPACE_META: ARCSTAT_INCR(arcstat_metadata_size, space); break; - case ARC_SPACE_OTHER: - ARCSTAT_INCR(arcstat_other_size, space); + case ARC_SPACE_BONUS: + ARCSTAT_INCR(arcstat_bonus_size, space); + break; + case ARC_SPACE_DNODE: + ARCSTAT_INCR(arcstat_dnode_size, space); + break; + case ARC_SPACE_DBUF: + ARCSTAT_INCR(arcstat_dbuf_size, space); break; case ARC_SPACE_HDRS: ARCSTAT_INCR(arcstat_hdr_size, space); @@ -1707,8 +1732,14 @@ arc_space_return(uint64_t space, arc_space_type_t type) case ARC_SPACE_META: ARCSTAT_INCR(arcstat_metadata_size, -space); break; - case ARC_SPACE_OTHER: - ARCSTAT_INCR(arcstat_other_size, -space); + case ARC_SPACE_BONUS: + ARCSTAT_INCR(arcstat_bonus_size, -space); + break; + case ARC_SPACE_DNODE: + ARCSTAT_INCR(arcstat_dnode_size, -space); + break; + case ARC_SPACE_DBUF: + ARCSTAT_INCR(arcstat_dbuf_size, -space); break; case ARC_SPACE_HDRS: ARCSTAT_INCR(arcstat_hdr_size, -space); @@ -2595,6 +2626,18 @@ arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes, * we're evicting all available buffers. */ while (total_evicted < bytes || bytes == ARC_EVICT_ALL) { + int sublist_idx = multilist_get_random_index(ml); + uint64_t scan_evicted = 0; + + /* + * Try to reduce pinned dnodes with a floor of arc_dnode_limit. + * Request that 10% of the LRUs be scanned by the superblock + * shrinker. + */ + if (type == ARC_BUFC_DATA && arc_dnode_size > arc_dnode_limit) + arc_prune_async((arc_dnode_size - arc_dnode_limit) / + sizeof (dnode_t) / zfs_arc_dnode_reduce_percent); + /* * Start eviction using a randomly selected sublist, * this is to try and evenly balance eviction across all @@ -2602,9 +2645,6 @@ arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes, * (e.g. index 0) would cause evictions to favor certain * sublists over others. */ - int sublist_idx = multilist_get_random_index(ml); - uint64_t scan_evicted = 0; - for (i = 0; i < num_sublists; i++) { uint64_t bytes_remaining; uint64_t bytes_evicted; @@ -2707,12 +2747,7 @@ arc_prune_task(void *ptr) if (func != NULL) func(ap->p_adjust, ap->p_private); - /* Callback unregistered concurrently with execution */ - if (refcount_remove(&ap->p_refcnt, func) == 0) { - ASSERT(!list_link_active(&ap->p_node)); - refcount_destroy(&ap->p_refcnt); - kmem_free(ap, sizeof (*ap)); - } + refcount_remove(&ap->p_refcnt, func); } /* @@ -4628,13 +4663,19 @@ arc_add_prune_callback(arc_prune_func_t *func, void *private) void arc_remove_prune_callback(arc_prune_t *p) { + boolean_t wait = B_FALSE; mutex_enter(&arc_prune_mtx); list_remove(&arc_prune_list, p); - if (refcount_remove(&p->p_refcnt, &arc_prune_list) == 0) { - refcount_destroy(&p->p_refcnt); - kmem_free(p, sizeof (*p)); - } + if (refcount_remove(&p->p_refcnt, &arc_prune_list) > 0) + wait = B_TRUE; mutex_exit(&arc_prune_mtx); + + /* wait for arc_prune_task to finish */ + if (wait) + taskq_wait_outstanding(arc_prune_taskq, 0); + ASSERT0(refcount_count(&p->p_refcnt)); + refcount_destroy(&p->p_refcnt); + kmem_free(p, sizeof (*p)); } void @@ -4980,6 +5021,15 @@ arc_write_ready(zio_t *zio) hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS; } +static void +arc_write_children_ready(zio_t *zio) +{ + arc_write_callback_t *callback = zio->io_private; + arc_buf_t *buf = callback->awcb_buf; + + callback->awcb_children_ready(zio, buf, callback->awcb_private); +} + /* * The SPA calls this callback for each physical write that happens on behalf * of a logical write. See the comment in dbuf_write_physdone() for details. @@ -5076,7 +5126,8 @@ arc_write_done(zio_t *zio) zio_t * arc_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, arc_buf_t *buf, boolean_t l2arc, boolean_t l2arc_compress, - const zio_prop_t *zp, arc_done_func_t *ready, arc_done_func_t *physdone, + const zio_prop_t *zp, arc_done_func_t *ready, + arc_done_func_t *children_ready, arc_done_func_t *physdone, arc_done_func_t *done, void *private, zio_priority_t priority, int zio_flags, const zbookmark_phys_t *zb) { @@ -5096,13 +5147,16 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg, hdr->b_flags |= ARC_FLAG_L2COMPRESS; callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); callback->awcb_ready = ready; + callback->awcb_children_ready = children_ready; callback->awcb_physdone = physdone; callback->awcb_done = done; callback->awcb_private = private; callback->awcb_buf = buf; zio = zio_write(pio, spa, txg, bp, buf->b_data, hdr->b_size, zp, - arc_write_ready, arc_write_physdone, arc_write_done, callback, + arc_write_ready, + (children_ready != NULL) ? arc_write_children_ready : NULL, + arc_write_physdone, arc_write_done, callback, priority, zio_flags, zb); return (zio); @@ -5311,6 +5365,7 @@ arc_tuning_update(void) arc_c = arc_c_max; arc_p = (arc_c >> 1); arc_meta_limit = MIN(arc_meta_limit, (3 * arc_c_max) / 4); + arc_dnode_limit = arc_meta_limit / 10; } /* Valid range: 32M - */ @@ -5327,6 +5382,7 @@ arc_tuning_update(void) (zfs_arc_meta_min <= arc_c_max)) { arc_meta_min = zfs_arc_meta_min; arc_meta_limit = MAX(arc_meta_limit, arc_meta_min); + arc_dnode_limit = arc_meta_limit / 10; } /* Valid range: - */ @@ -5335,6 +5391,12 @@ arc_tuning_update(void) (zfs_arc_meta_limit <= arc_c_max)) arc_meta_limit = zfs_arc_meta_limit; + /* Valid range: - */ + if ((zfs_arc_dnode_limit) && (zfs_arc_dnode_limit != arc_dnode_limit) && + (zfs_arc_dnode_limit >= zfs_arc_meta_min) && + (zfs_arc_dnode_limit <= arc_c_max)) + arc_dnode_limit = zfs_arc_dnode_limit; + /* Valid range: 1 - N */ if (zfs_arc_grow_retry) arc_grow_retry = zfs_arc_grow_retry; @@ -5433,6 +5495,8 @@ arc_init(void) arc_meta_max = 0; /* Set limit to 3/4 of arc_c_max with a floor of arc_meta_min */ arc_meta_limit = MAX((3 * arc_c_max) / 4, arc_meta_min); + /* Default dnode limit is 10% of overall meta limit */ + arc_dnode_limit = arc_meta_limit / 10; /* Apply user specified tunings */ arc_tuning_update(); @@ -7186,4 +7250,11 @@ MODULE_PARM_DESC(zfs_arc_lotsfree_percent, module_param(zfs_arc_sys_free, ulong, 0644); MODULE_PARM_DESC(zfs_arc_sys_free, "System free memory target size in bytes"); +module_param(zfs_arc_dnode_limit, ulong, 0644); +MODULE_PARM_DESC(zfs_arc_dnode_limit, "Minimum bytes of dnodes in arc"); + +module_param(zfs_arc_dnode_reduce_percent, ulong, 0644); +MODULE_PARM_DESC(zfs_arc_dnode_reduce_percent, + "Percentage of excess dnodes to try to unpin"); + #endif diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c index fd51d59a8edb..af2f20d63b5c 100644 --- a/module/zfs/dbuf.c +++ b/module/zfs/dbuf.c @@ -478,7 +478,6 @@ dbuf_verify(dmu_buf_impl_t *db) ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); } else if (db->db_blkid == DMU_SPILL_BLKID) { ASSERT(dn != NULL); - ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); ASSERT0(db->db.db_offset); } else { ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); @@ -543,13 +542,50 @@ dbuf_verify(dmu_buf_impl_t *db) * If the blkptr isn't set but they have nonzero data, * it had better be dirty, otherwise we'll lose that * data when we evict this buffer. + * + * There is an exception to this rule for indirect blocks; in + * this case, if the indirect block is a hole, we fill in a few + * fields on each of the child blocks (importantly, birth time) + * to prevent hole birth times from being lost when you + * partially fill in a hole. */ if (db->db_dirtycnt == 0) { - ASSERTV(uint64_t *buf = db->db.db_data); - int i; + if (db->db_level == 0) { + uint64_t *buf = db->db.db_data; + int i; - for (i = 0; i < db->db.db_size >> 3; i++) { - ASSERT(buf[i] == 0); + for (i = 0; i < db->db.db_size >> 3; i++) { + ASSERT(buf[i] == 0); + } + } else { + int i; + blkptr_t *bps = db->db.db_data; + ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==, + db->db.db_size); + /* + * We want to verify that all the blkptrs in the + * indirect block are holes, but we may have + * automatically set up a few fields for them. + * We iterate through each blkptr and verify + * they only have those fields set. + */ + for (i = 0; + i < db->db.db_size / sizeof (blkptr_t); + i++) { + blkptr_t *bp = &bps[i]; + ASSERT(ZIO_CHECKSUM_IS_ZERO( + &bp->blk_cksum)); + ASSERT( + DVA_IS_EMPTY(&bp->blk_dva[0]) && + DVA_IS_EMPTY(&bp->blk_dva[1]) && + DVA_IS_EMPTY(&bp->blk_dva[2])); + ASSERT0(bp->blk_fill); + ASSERT0(bp->blk_pad[0]); + ASSERT0(bp->blk_pad[1]); + ASSERT(!BP_IS_EMBEDDED(bp)); + ASSERT(BP_IS_HOLE(bp)); + ASSERT0(bp->blk_phys_birth); + } } } } @@ -693,13 +729,18 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) ASSERT(db->db_buf == NULL); if (db->db_blkid == DMU_BONUS_BLKID) { + /* + * The bonus length stored in the dnode may be less than + * the maximum available space in the bonus buffer. + */ int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); + int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); ASSERT3U(bonuslen, <=, db->db.db_size); - db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); - arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); - if (bonuslen < DN_MAX_BONUSLEN) - bzero(db->db.db_data, DN_MAX_BONUSLEN); + db->db.db_data = zio_buf_alloc(max_bonuslen); + arc_space_consume(max_bonuslen, ARC_SPACE_BONUS); + if (bonuslen < max_bonuslen) + bzero(db->db.db_data, max_bonuslen); if (bonuslen) bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); DB_DNODE_EXIT(db); @@ -718,10 +759,32 @@ dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) BP_IS_HOLE(db->db_blkptr)))) { arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); - DB_DNODE_EXIT(db); dbuf_set_data(db, arc_buf_alloc(db->db_objset->os_spa, db->db.db_size, db, type)); bzero(db->db.db_data, db->db.db_size); + + if (db->db_blkptr != NULL && db->db_level > 0 && + BP_IS_HOLE(db->db_blkptr) && + db->db_blkptr->blk_birth != 0) { + blkptr_t *bps = db->db.db_data; + int i; + for (i = 0; i < ((1 << + DB_DNODE(db)->dn_indblkshift) / sizeof (blkptr_t)); + i++) { + blkptr_t *bp = &bps[i]; + ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, + 1 << dn->dn_indblkshift); + BP_SET_LSIZE(bp, + BP_GET_LEVEL(db->db_blkptr) == 1 ? + dn->dn_datablksz : + BP_GET_LSIZE(db->db_blkptr)); + BP_SET_TYPE(bp, BP_GET_TYPE(db->db_blkptr)); + BP_SET_LEVEL(bp, + BP_GET_LEVEL(db->db_blkptr) - 1); + BP_SET_BIRTH(bp, db->db_blkptr->blk_birth, 0); + } + } + DB_DNODE_EXIT(db); db->db_state = DB_CACHED; mutex_exit(&db->db_mtx); return (0); @@ -903,9 +966,11 @@ dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) ASSERT(dr->dr_txg >= txg - 2); if (db->db_blkid == DMU_BONUS_BLKID) { /* Note that the data bufs here are zio_bufs */ - dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); - arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); - bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); + dnode_t *dn = DB_DNODE(db); + int bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); + dr->dt.dl.dr_data = zio_buf_alloc(bonuslen); + arc_space_consume(bonuslen, ARC_SPACE_BONUS); + bcopy(db->db.db_data, dr->dt.dl.dr_data, bonuslen); } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { int size = db->db.db_size; arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); @@ -1799,8 +1864,10 @@ dbuf_clear(dmu_buf_impl_t *db) if (db->db_state == DB_CACHED) { ASSERT(db->db.db_data != NULL); if (db->db_blkid == DMU_BONUS_BLKID) { - zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); - arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); + int slots = DB_DNODE(db)->dn_num_slots; + int bonuslen = DN_SLOTS_TO_BONUSLEN(slots); + zio_buf_free(db->db.db_data, bonuslen); + arc_space_return(bonuslen, ARC_SPACE_BONUS); } db->db.db_data = NULL; db->db_state = DB_UNCACHED; @@ -1870,7 +1937,7 @@ dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, mutex_enter(&dn->dn_mtx); if (dn->dn_have_spill && (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) - *bpp = &dn->dn_phys->dn_spill; + *bpp = DN_SPILL_BLKPTR(dn->dn_phys); else *bpp = NULL; dbuf_add_ref(dn->dn_dbuf, NULL); @@ -1959,13 +2026,13 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, if (blkid == DMU_BONUS_BLKID) { ASSERT3P(parent, ==, dn->dn_dbuf); - db->db.db_size = DN_MAX_BONUSLEN - + db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - (dn->dn_nblkptr-1) * sizeof (blkptr_t); ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); db->db.db_offset = DMU_BONUS_BLKID; db->db_state = DB_UNCACHED; /* the bonus dbuf is not placed in the hash table */ - arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); + arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF); return (db); } else if (blkid == DMU_SPILL_BLKID) { db->db.db_size = (blkptr != NULL) ? @@ -1999,7 +2066,7 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, dn->dn_unlisted_l0_blkid = db->db_blkid + 1; db->db_state = DB_UNCACHED; mutex_exit(&dn->dn_dbufs_mtx); - arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); + arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF); if (parent && parent != dn->dn_dbuf) dbuf_add_ref(parent, db); @@ -2076,7 +2143,7 @@ dbuf_destroy(dmu_buf_impl_t *db) ASSERT(db->db_data_pending == NULL); kmem_cache_free(dbuf_cache, db); - arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); + arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_DBUF); } typedef struct dbuf_prefetch_arg { @@ -2751,7 +2818,7 @@ dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) return; if (db->db_blkid == DMU_SPILL_BLKID) { - db->db_blkptr = &dn->dn_phys->dn_spill; + db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys); BP_ZERO(db->db_blkptr); return; } @@ -2876,6 +2943,22 @@ dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) if (db->db_blkid == DMU_SPILL_BLKID) { mutex_enter(&dn->dn_mtx); + if (!(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) { + /* + * In the previous transaction group, the bonus buffer + * was entirely used to store the attributes for the + * dnode which overrode the dn_spill field. However, + * when adding more attributes to the file a spill + * block was required to hold the extra attributes. + * + * Make sure to clear the garbage left in the dn_spill + * field from the previous attributes in the bonus + * buffer. Otherwise, after writing out the spill + * block to the new allocated dva, it will free + * the old block pointed to by the invalid dn_spill. + */ + db->db_blkptr = NULL; + } dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; mutex_exit(&dn->dn_mtx); } @@ -2891,13 +2974,16 @@ dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) ASSERT(*datap != NULL); ASSERT0(db->db_level); - ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); + ASSERT3U(dn->dn_phys->dn_bonuslen, <=, + DN_SLOTS_TO_BONUSLEN(dn->dn_phys->dn_extra_slots + 1)); bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); DB_DNODE_EXIT(db); if (*datap != db->db.db_data) { - zio_buf_free(*datap, DN_MAX_BONUSLEN); - arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); + int slots = DB_DNODE(db)->dn_num_slots; + int bonuslen = DN_SLOTS_TO_BONUSLEN(slots); + zio_buf_free(*datap, bonuslen); + arc_space_return(bonuslen, ARC_SPACE_BONUS); } db->db_data_pending = NULL; drp = &db->db_last_dirty; @@ -3048,7 +3134,7 @@ dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) if (db->db_blkid == DMU_SPILL_BLKID) { ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); ASSERT(!(BP_IS_HOLE(bp)) && - db->db_blkptr == &dn->dn_phys->dn_spill); + db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); } #endif @@ -3060,11 +3146,16 @@ dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) mutex_exit(&dn->dn_mtx); if (dn->dn_type == DMU_OT_DNODE) { - dnode_phys_t *dnp = db->db.db_data; - for (i = db->db.db_size >> DNODE_SHIFT; i > 0; - i--, dnp++) { - if (dnp->dn_type != DMU_OT_NONE) + i = 0; + while (i < db->db.db_size) { + dnode_phys_t *dnp = db->db.db_data + i; + + i += DNODE_MIN_SIZE; + if (dnp->dn_type != DMU_OT_NONE) { fill++; + i += dnp->dn_extra_slots * + DNODE_MIN_SIZE; + } } } else { if (BP_IS_HOLE(bp)) { @@ -3094,6 +3185,45 @@ dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) rw_exit(&dn->dn_struct_rwlock); } +/* ARGSUSED */ +/* + * This function gets called just prior to running through the compression + * stage of the zio pipeline. If we're an indirect block comprised of only + * holes, then we want this indirect to be compressed away to a hole. In + * order to do that we must zero out any information about the holes that + * this indirect points to prior to before we try to compress it. + */ +static void +dbuf_write_children_ready(zio_t *zio, arc_buf_t *buf, void *vdb) +{ + dmu_buf_impl_t *db = vdb; + dnode_t *dn; + blkptr_t *bp; + uint64_t i; + int epbs; + + ASSERT3U(db->db_level, >, 0); + DB_DNODE_ENTER(db); + dn = DB_DNODE(db); + epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; + + /* Determine if all our children are holes */ + for (i = 0, bp = db->db.db_data; i < 1 << epbs; i++, bp++) { + if (!BP_IS_HOLE(bp)) + break; + } + + /* + * If all the children are holes, then zero them all out so that + * we may get compressed away. + */ + if (i == 1 << epbs) { + /* didn't find any non-holes */ + bzero(db->db.db_data, db->db.db_size); + } + DB_DNODE_EXIT(db); +} + /* * The SPA will call this callback several times for each zio - once * for every physical child i/o (zio->io_phys_children times). This @@ -3172,7 +3302,7 @@ dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) dn = DB_DNODE(db); ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && - db->db_blkptr == &dn->dn_phys->dn_spill); + db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys)); DB_DNODE_EXIT(db); } #endif @@ -3348,7 +3478,8 @@ dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) dr->dr_zio = zio_write(zio, os->os_spa, txg, &dr->dr_bp_copy, contents, db->db.db_size, &zp, - dbuf_write_override_ready, NULL, dbuf_write_override_done, + dbuf_write_override_ready, NULL, NULL, + dbuf_write_override_done, dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); mutex_enter(&db->db_mtx); dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; @@ -3359,14 +3490,26 @@ dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF); dr->dr_zio = zio_write(zio, os->os_spa, txg, &dr->dr_bp_copy, NULL, db->db.db_size, &zp, - dbuf_write_nofill_ready, NULL, dbuf_write_nofill_done, db, + dbuf_write_nofill_ready, NULL, NULL, + dbuf_write_nofill_done, db, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); } else { + arc_done_func_t *children_ready_cb = NULL; ASSERT(arc_released(data)); + + /* + * For indirect blocks, we want to setup the children + * ready callback so that we can properly handle an indirect + * block that only contains holes. + */ + if (db->db_level != 0) + children_ready_cb = dbuf_write_children_ready; + dr->dr_zio = arc_write(zio, os->os_spa, txg, &dr->dr_bp_copy, data, DBUF_IS_L2CACHEABLE(db), DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready, + children_ready_cb, dbuf_write_physdone, dbuf_write_done, db, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); } diff --git a/module/zfs/dbuf_stats.c b/module/zfs/dbuf_stats.c index afdf828ed542..6f39f80e563a 100644 --- a/module/zfs/dbuf_stats.c +++ b/module/zfs/dbuf_stats.c @@ -148,7 +148,6 @@ dbuf_stats_hash_table_data(char *buf, size_t size, void *data) } mutex_enter(&db->db_mtx); - mutex_exit(DBUF_HASH_MUTEX(h, dsh->idx)); if (db->db_state != DB_EVICTING) { length = __dbuf_stats_hash_table_data(buf, size, db); @@ -157,7 +156,6 @@ dbuf_stats_hash_table_data(char *buf, size_t size, void *data) } mutex_exit(&db->db_mtx); - mutex_enter(DBUF_HASH_MUTEX(h, dsh->idx)); } mutex_exit(DBUF_HASH_MUTEX(h, dsh->idx)); diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index 786287834183..e1dfb41ff353 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -20,8 +20,9 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2015 by Delphix. All rights reserved. + * Copyright (c) 2011, 2016 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. + * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2014, Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2015 by Chunwei Chen. All rights reserved. */ @@ -179,7 +180,7 @@ dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset, int dmu_bonus_max(void) { - return (DN_MAX_BONUSLEN); + return (DN_OLD_MAX_BONUSLEN); } int @@ -1480,10 +1481,11 @@ dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd, dsa->dsa_zgd = zgd; dsa->dsa_tx = tx; - zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp, - zgd->zgd_db->db_data, zgd->zgd_db->db_size, zp, - dmu_sync_late_arrival_ready, NULL, dmu_sync_late_arrival_done, dsa, - ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL|ZIO_FLAG_FASTWRITE, zb)); + zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), + zgd->zgd_bp, zgd->zgd_db->db_data, zgd->zgd_db->db_size, + zp, dmu_sync_late_arrival_ready, NULL, + NULL, dmu_sync_late_arrival_done, dsa, ZIO_PRIORITY_SYNC_WRITE, + ZIO_FLAG_CANFAIL, zb)); return (0); } @@ -1636,8 +1638,8 @@ dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd) zio_nowait(arc_write(pio, os->os_spa, txg, bp, dr->dt.dl.dr_data, DBUF_IS_L2CACHEABLE(db), DBUF_IS_L2COMPRESSIBLE(db), &zp, dmu_sync_ready, - NULL, dmu_sync_done, dsa, ZIO_PRIORITY_SYNC_WRITE, - ZIO_FLAG_CANFAIL, &zb)); + NULL, NULL, dmu_sync_done, dsa, + ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, &zb)); return (0); } @@ -1851,6 +1853,7 @@ __dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi) doi->doi_type = dn->dn_type; doi->doi_bonus_type = dn->dn_bonustype; doi->doi_bonus_size = dn->dn_bonuslen; + doi->doi_dnodesize = dn->dn_num_slots << DNODE_SHIFT; doi->doi_indirection = dn->dn_nlevels; doi->doi_checksum = dn->dn_checksum; doi->doi_compress = dn->dn_compress; @@ -1922,9 +1925,21 @@ dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize, dn = DB_DNODE(db); *blksize = dn->dn_datablksz; - /* add 1 for dnode space */ + /* add in number of slots used for the dnode itself */ *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >> - SPA_MINBLOCKSHIFT) + 1; + SPA_MINBLOCKSHIFT) + dn->dn_num_slots; + DB_DNODE_EXIT(db); +} + +void +dmu_object_dnsize_from_db(dmu_buf_t *db_fake, int *dnsize) +{ + dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; + dnode_t *dn; + + DB_DNODE_ENTER(db); + dn = DB_DNODE(db); + *dnsize = dn->dn_num_slots << DNODE_SHIFT; DB_DNODE_EXIT(db); } @@ -2018,6 +2033,7 @@ EXPORT_SYMBOL(dmu_object_info); EXPORT_SYMBOL(dmu_object_info_from_dnode); EXPORT_SYMBOL(dmu_object_info_from_db); EXPORT_SYMBOL(dmu_object_size_from_db); +EXPORT_SYMBOL(dmu_object_dnsize_from_db); EXPORT_SYMBOL(dmu_object_set_blocksize); EXPORT_SYMBOL(dmu_object_set_checksum); EXPORT_SYMBOL(dmu_object_set_compress); diff --git a/module/zfs/dmu_object.c b/module/zfs/dmu_object.c index 5faecafc7d86..e54043fc3e3a 100644 --- a/module/zfs/dmu_object.c +++ b/module/zfs/dmu_object.c @@ -30,26 +30,55 @@ #include #include #include +#include uint64_t dmu_object_alloc(objset_t *os, dmu_object_type_t ot, int blocksize, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) +{ + return dmu_object_alloc_dnsize(os, ot, blocksize, bonustype, bonuslen, + 0, tx); +} + +uint64_t +dmu_object_alloc_dnsize(objset_t *os, dmu_object_type_t ot, int blocksize, + dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx) { uint64_t object; - uint64_t L2_dnode_count = DNODES_PER_BLOCK << + uint64_t L1_dnode_count = DNODES_PER_BLOCK << (DMU_META_DNODE(os)->dn_indblkshift - SPA_BLKPTRSHIFT); dnode_t *dn = NULL; - int restarted = B_FALSE; + int dn_slots = dnodesize >> DNODE_SHIFT; + boolean_t restarted = B_FALSE; + + if (dn_slots == 0) { + dn_slots = DNODE_MIN_SLOTS; + } else { + ASSERT3S(dn_slots, >=, DNODE_MIN_SLOTS); + ASSERT3S(dn_slots, <=, DNODE_MAX_SLOTS); + } mutex_enter(&os->os_obj_lock); for (;;) { object = os->os_obj_next; /* - * Each time we polish off an L2 bp worth of dnodes - * (2^13 objects), move to another L2 bp that's still - * reasonably sparse (at most 1/4 full). Look from the - * beginning once, but after that keep looking from here. - * If we can't find one, just keep going from here. + * Each time we polish off a L1 bp worth of dnodes (2^12 + * objects), move to another L1 bp that's still + * reasonably sparse (at most 1/4 full). Look from the + * beginning at most once per txg. If we still can't + * allocate from that L1 block, search for an empty L0 + * block, which will quickly skip to the end of the + * metadnode if the no nearby L0 blocks are empty. This + * fallback avoids a pathology where full dnode blocks + * containing large dnodes appear sparse because they + * have a low blk_fill, leading to many failed + * allocation attempts. In the long term a better + * mechanism to search for sparse metadnode regions, + * such as spacemaps, could be implemented. + * + * os_scan_dnodes is set during txg sync if enough objects + * have been freed since the previous rescan to justify + * backfilling again. * * Note that dmu_traverse depends on the behavior that we use * multiple blocks of the dnode object before going back to @@ -57,16 +86,26 @@ dmu_object_alloc(objset_t *os, dmu_object_type_t ot, int blocksize, * that property or find another solution to the issues * described in traverse_visitbp. */ - if (P2PHASE(object, L2_dnode_count) == 0) { - uint64_t offset = restarted ? object << DNODE_SHIFT : 0; - int error = dnode_next_offset(DMU_META_DNODE(os), - DNODE_FIND_HOLE, - &offset, 2, DNODES_PER_BLOCK >> 2, 0); + if (P2PHASE(object, L1_dnode_count) == 0) { + uint64_t offset; + uint64_t blkfill; + int minlvl; + int error; + if (os->os_rescan_dnodes) { + offset = 0; + os->os_rescan_dnodes = B_FALSE; + } else { + offset = object << DNODE_SHIFT; + } + blkfill = restarted ? 1 : DNODES_PER_BLOCK >> 2; + minlvl = restarted ? 1 : 2; restarted = B_TRUE; + error = dnode_next_offset(DMU_META_DNODE(os), + DNODE_FIND_HOLE, &offset, minlvl, blkfill, 0); if (error == 0) object = offset >> DNODE_SHIFT; } - os->os_obj_next = ++object; + os->os_obj_next = object + dn_slots; /* * XXX We should check for an i/o error here and return @@ -74,16 +113,22 @@ dmu_object_alloc(objset_t *os, dmu_object_type_t ot, int blocksize, * dmu_tx_assign(), but there is currently no mechanism * to do so. */ - (void) dnode_hold_impl(os, object, DNODE_MUST_BE_FREE, + (void) dnode_hold_impl(os, object, DNODE_MUST_BE_FREE, dn_slots, FTAG, &dn); if (dn) break; if (dmu_object_next(os, &object, B_TRUE, 0) == 0) - os->os_obj_next = object - 1; + os->os_obj_next = object; + else + /* + * Skip to next known valid starting point for a dnode. + */ + os->os_obj_next = P2ROUNDUP(object + 1, + DNODES_PER_BLOCK); } - dnode_allocate(dn, ot, blocksize, 0, bonustype, bonuslen, tx); + dnode_allocate(dn, ot, blocksize, 0, bonustype, bonuslen, dn_slots, tx); dnode_rele(dn, FTAG); mutex_exit(&os->os_obj_lock); @@ -95,17 +140,34 @@ dmu_object_alloc(objset_t *os, dmu_object_type_t ot, int blocksize, int dmu_object_claim(objset_t *os, uint64_t object, dmu_object_type_t ot, int blocksize, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) +{ + return (dmu_object_claim_dnsize(os, object, ot, blocksize, bonustype, + bonuslen, 0, tx)); +} + +int +dmu_object_claim_dnsize(objset_t *os, uint64_t object, dmu_object_type_t ot, + int blocksize, dmu_object_type_t bonustype, int bonuslen, + int dnodesize, dmu_tx_t *tx) { dnode_t *dn; + int dn_slots = dnodesize >> DNODE_SHIFT; int err; + if (dn_slots == 0) + dn_slots = DNODE_MIN_SLOTS; + ASSERT3S(dn_slots, >=, DNODE_MIN_SLOTS); + ASSERT3S(dn_slots, <=, DNODE_MAX_SLOTS); + if (object == DMU_META_DNODE_OBJECT && !dmu_tx_private_ok(tx)) return (SET_ERROR(EBADF)); - err = dnode_hold_impl(os, object, DNODE_MUST_BE_FREE, FTAG, &dn); + err = dnode_hold_impl(os, object, DNODE_MUST_BE_FREE, dn_slots, + FTAG, &dn); if (err) return (err); - dnode_allocate(dn, ot, blocksize, 0, bonustype, bonuslen, tx); + + dnode_allocate(dn, ot, blocksize, 0, bonustype, bonuslen, dn_slots, tx); dnode_rele(dn, FTAG); dmu_tx_add_new_object(tx, os, object); @@ -115,24 +177,35 @@ dmu_object_claim(objset_t *os, uint64_t object, dmu_object_type_t ot, int dmu_object_reclaim(objset_t *os, uint64_t object, dmu_object_type_t ot, int blocksize, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) +{ + return (dmu_object_reclaim_dnsize(os, object, ot, blocksize, bonustype, + bonuslen, 0, tx)); +} + +int +dmu_object_reclaim_dnsize(objset_t *os, uint64_t object, dmu_object_type_t ot, + int blocksize, dmu_object_type_t bonustype, int bonuslen, int dnodesize, + dmu_tx_t *tx) { dnode_t *dn; + int dn_slots = dnodesize >> DNODE_SHIFT; int err; if (object == DMU_META_DNODE_OBJECT) return (SET_ERROR(EBADF)); - err = dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, + err = dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, 0, FTAG, &dn); if (err) return (err); - dnode_reallocate(dn, ot, blocksize, bonustype, bonuslen, tx); + dnode_reallocate(dn, ot, blocksize, bonustype, bonuslen, dn_slots, tx); dnode_rele(dn, FTAG); return (err); } + int dmu_object_free(objset_t *os, uint64_t object, dmu_tx_t *tx) { @@ -141,7 +214,7 @@ dmu_object_free(objset_t *os, uint64_t object, dmu_tx_t *tx) ASSERT(object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx)); - err = dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, + err = dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, 0, FTAG, &dn); if (err) return (err); @@ -162,9 +235,30 @@ dmu_object_free(objset_t *os, uint64_t object, dmu_tx_t *tx) int dmu_object_next(objset_t *os, uint64_t *objectp, boolean_t hole, uint64_t txg) { - uint64_t offset = (*objectp + 1) << DNODE_SHIFT; + uint64_t offset; + dmu_object_info_t doi; + struct dsl_dataset *ds = os->os_dsl_dataset; + int dnodesize; int error; + /* + * Avoid expensive dnode hold if this dataset doesn't use large dnodes. + */ + if (ds && ds->ds_feature_inuse[SPA_FEATURE_LARGE_DNODE]) { + error = dmu_object_info(os, *objectp, &doi); + if (error && !(error == EINVAL && *objectp == 0)) + return (SET_ERROR(error)); + else + dnodesize = doi.doi_dnodesize; + } else { + dnodesize = DNODE_MIN_SIZE; + } + + if (*objectp == 0) + offset = 1 << DNODE_SHIFT; + else + offset = (*objectp << DNODE_SHIFT) + dnodesize; + error = dnode_next_offset(DMU_META_DNODE(os), (hole ? DNODE_FIND_HOLE : 0), &offset, 0, DNODES_PER_BLOCK, txg); @@ -226,8 +320,11 @@ dmu_object_free_zapified(objset_t *mos, uint64_t object, dmu_tx_t *tx) #if defined(_KERNEL) && defined(HAVE_SPL) EXPORT_SYMBOL(dmu_object_alloc); +EXPORT_SYMBOL(dmu_object_alloc_dnsize); EXPORT_SYMBOL(dmu_object_claim); +EXPORT_SYMBOL(dmu_object_claim_dnsize); EXPORT_SYMBOL(dmu_object_reclaim); +EXPORT_SYMBOL(dmu_object_reclaim_dnsize); EXPORT_SYMBOL(dmu_object_free); EXPORT_SYMBOL(dmu_object_next); EXPORT_SYMBOL(dmu_object_zapify); diff --git a/module/zfs/dmu_objset.c b/module/zfs/dmu_objset.c index f9c534eb5736..22ca84d96a27 100644 --- a/module/zfs/dmu_objset.c +++ b/module/zfs/dmu_objset.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. @@ -52,6 +52,7 @@ #include #include #include +#include /* * Needed to close a window in dnode_move() that allows the objset to be freed @@ -67,6 +68,13 @@ krwlock_t os_lock; */ int dmu_find_threads = 0; +/* + * Backfill lower metadnode objects after this many have been freed. + * Backfilling negatively impacts object creation rates, so only do it + * if there are enough holes to fill. + */ +int dmu_rescan_dnode_threshold = 1 << DN_MAX_INDBLKSHIFT; + static void dmu_objset_find_dp_cb(void *arg); void @@ -130,6 +138,12 @@ dmu_objset_id(objset_t *os) return (ds ? ds->ds_object : 0); } +uint64_t +dmu_objset_dnodesize(objset_t *os) +{ + return (os->os_dnodesize); +} + zfs_sync_type_t dmu_objset_syncprop(objset_t *os) { @@ -259,6 +273,34 @@ redundant_metadata_changed_cb(void *arg, uint64_t newval) os->os_redundant_metadata = newval; } +static void +dnodesize_changed_cb(void *arg, uint64_t newval) +{ + objset_t *os = arg; + + switch (newval) { + case ZFS_DNSIZE_LEGACY: + os->os_dnodesize = DNODE_MIN_SIZE; + break; + case ZFS_DNSIZE_AUTO: + /* + * Choose a dnode size that will work well for most + * workloads if the user specified "auto". Future code + * improvements could dynamically select a dnode size + * based on observed workload patterns. + */ + os->os_dnodesize = DNODE_MIN_SIZE * 2; + break; + case ZFS_DNSIZE_1K: + case ZFS_DNSIZE_2K: + case ZFS_DNSIZE_4K: + case ZFS_DNSIZE_8K: + case ZFS_DNSIZE_16K: + os->os_dnodesize = newval; + break; + } +} + static void logbias_changed_cb(void *arg, uint64_t newval) { @@ -363,6 +405,17 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp, * checksum/compression/copies. */ if (ds != NULL) { + boolean_t needlock = B_FALSE; + + /* + * Note: it's valid to open the objset if the dataset is + * long-held, in which case the pool_config lock will not + * be held. + */ + if (!dsl_pool_config_held(dmu_objset_pool(os))) { + needlock = B_TRUE; + dsl_pool_config_enter(dmu_objset_pool(os), FTAG); + } err = dsl_prop_register(ds, zfs_prop_to_name(ZFS_PROP_PRIMARYCACHE), primary_cache_changed_cb, os); @@ -413,7 +466,14 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp, zfs_prop_to_name(ZFS_PROP_RECORDSIZE), recordsize_changed_cb, os); } + if (err == 0) { + err = dsl_prop_register(ds, + zfs_prop_to_name(ZFS_PROP_DNODESIZE), + dnodesize_changed_cb, os); + } } + if (needlock) + dsl_pool_config_exit(dmu_objset_pool(os), FTAG); if (err != 0) { VERIFY(arc_buf_remove_ref(os->os_phys_buf, &os->os_phys_buf)); @@ -431,6 +491,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp, os->os_sync = ZFS_SYNC_STANDARD; os->os_primary_cache = ZFS_CACHE_ALL; os->os_secondary_cache = ZFS_CACHE_ALL; + os->os_dnodesize = DNODE_MIN_SIZE; } if (ds == NULL || !ds->ds_is_snapshot) @@ -472,6 +533,13 @@ dmu_objset_from_ds(dsl_dataset_t *ds, objset_t **osp) { int err = 0; + /* + * We shouldn't be doing anything with dsl_dataset_t's unless the + * pool_config lock is held, or the dataset is long-held. + */ + ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool) || + dsl_dataset_long_held(ds)); + mutex_enter(&ds->ds_opening_lock); if (ds->ds_objset == NULL) { objset_t *os; @@ -603,7 +671,7 @@ dmu_objset_refresh_ownership(objset_t *os, void *tag) { dsl_pool_t *dp; dsl_dataset_t *ds, *newds; - char name[MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; ds = os->os_dsl_dataset; VERIFY3P(ds, !=, NULL); @@ -760,8 +828,8 @@ dmu_objset_create_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp, mdn = DMU_META_DNODE(os); - dnode_allocate(mdn, DMU_OT_DNODE, 1 << DNODE_BLOCK_SHIFT, - DN_MAX_INDBLKSHIFT, DMU_OT_NONE, 0, tx); + dnode_allocate(mdn, DMU_OT_DNODE, DNODE_BLOCK_SIZE, DN_MAX_INDBLKSHIFT, + DMU_OT_NONE, 0, DNODE_MIN_SLOTS, tx); /* * We don't want to have to increase the meta-dnode's nlevels @@ -827,6 +895,9 @@ dmu_objset_create_check(void *arg, dmu_tx_t *tx) if (strchr(doca->doca_name, '@') != NULL) return (SET_ERROR(EINVAL)); + if (strlen(doca->doca_name) >= ZFS_MAX_DATASET_NAME_LEN) + return (SET_ERROR(ENAMETOOLONG)); + error = dsl_dir_hold(dp, doca->doca_name, FTAG, &pdd, &tail); if (error != 0) return (error); @@ -913,6 +984,9 @@ dmu_objset_clone_check(void *arg, dmu_tx_t *tx) if (strchr(doca->doca_clone, '@') != NULL) return (SET_ERROR(EINVAL)); + if (strlen(doca->doca_clone) >= ZFS_MAX_DATASET_NAME_LEN) + return (SET_ERROR(ENAMETOOLONG)); + error = dsl_dir_hold(dp, doca->doca_clone, FTAG, &pdd, &tail); if (error != 0) return (error); @@ -952,7 +1026,7 @@ dmu_objset_clone_sync(void *arg, dmu_tx_t *tx) const char *tail; dsl_dataset_t *origin, *ds; uint64_t obj; - char namebuf[MAXNAMELEN]; + char namebuf[ZFS_MAX_DATASET_NAME_LEN]; VERIFY0(dsl_dir_hold(dp, doca->doca_clone, FTAG, &pdd, &tail)); VERIFY0(dsl_dataset_hold(dp, doca->doca_origin, FTAG, &origin)); @@ -1109,9 +1183,9 @@ dmu_objset_sync(objset_t *os, zio_t *pio, dmu_tx_t *tx) zio = arc_write(pio, os->os_spa, tx->tx_txg, os->os_rootbp, os->os_phys_buf, DMU_OS_IS_L2CACHEABLE(os), - DMU_OS_IS_L2COMPRESSIBLE(os), &zp, dmu_objset_write_ready, - NULL, dmu_objset_write_done, os, ZIO_PRIORITY_ASYNC_WRITE, - ZIO_FLAG_MUSTSUCCEED, &zb); + DMU_OS_IS_L2COMPRESSIBLE(os), + &zp, dmu_objset_write_ready, NULL, NULL, dmu_objset_write_done, + os, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); /* * Sync special dnodes - the parent IO for the sync is the root block @@ -1151,6 +1225,13 @@ dmu_objset_sync(objset_t *os, zio_t *pio, dmu_tx_t *tx) if (dr->dr_zio) zio_nowait(dr->dr_zio); } + + /* Enable dnode backfill if enough objects have been freed. */ + if (os->os_freed_dnodes >= dmu_rescan_dnode_threshold) { + os->os_rescan_dnodes = B_TRUE; + os->os_freed_dnodes = 0; + } + /* * Free intent log blocks up to this tx. */ @@ -1187,7 +1268,7 @@ do_userquota_update(objset_t *os, uint64_t used, uint64_t flags, uint64_t user, uint64_t group, boolean_t subtract, dmu_tx_t *tx) { if ((flags & DNODE_FLAG_USERUSED_ACCOUNTED)) { - int64_t delta = DNODE_SIZE + used; + int64_t delta = DNODE_MIN_SIZE + used; if (subtract) delta = -delta; VERIFY3U(0, ==, zap_increment_int(os, DMU_USERUSED_OBJECT, @@ -1972,7 +2053,7 @@ dmu_objset_get_user(objset_t *os) /* * Determine name of filesystem, given name of snapshot. - * buf must be at least MAXNAMELEN bytes + * buf must be at least ZFS_MAX_DATASET_NAME_LEN bytes */ int dmu_fsname(const char *snapname, char *buf) @@ -1980,7 +2061,7 @@ dmu_fsname(const char *snapname, char *buf) char *atp = strchr(snapname, '@'); if (atp == NULL) return (SET_ERROR(EINVAL)); - if (atp - snapname >= MAXNAMELEN) + if (atp - snapname >= ZFS_MAX_DATASET_NAME_LEN) return (SET_ERROR(ENAMETOOLONG)); (void) strlcpy(buf, snapname, atp - snapname + 1); return (0); @@ -2008,6 +2089,7 @@ EXPORT_SYMBOL(dmu_objset_find); EXPORT_SYMBOL(dmu_objset_byteswap); EXPORT_SYMBOL(dmu_objset_evict_dbufs); EXPORT_SYMBOL(dmu_objset_snap_cmtime); +EXPORT_SYMBOL(dmu_objset_dnodesize); EXPORT_SYMBOL(dmu_objset_sync); EXPORT_SYMBOL(dmu_objset_is_dirty); diff --git a/module/zfs/dmu_send.c b/module/zfs/dmu_send.c index 7dc62dc208d5..80f7dc1aae11 100644 --- a/module/zfs/dmu_send.c +++ b/module/zfs/dmu_send.c @@ -20,10 +20,11 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2015 by Delphix. All rights reserved. * Copyright 2011 Nexenta Systems, Inc. All rights reserved. + * Copyright (c) 2011, 2015 by Delphix. All rights reserved. * Copyright (c) 2014, Joyent, Inc. All rights reserved. - * Copyright (c) 2011, 2014 by Delphix. All rights reserved. + * Copyright 2014 HybridCluster. All rights reserved. + * Copyright 2016 RackTop Systems. * Copyright (c) 2016 Actifio, Inc. All rights reserved. */ @@ -56,19 +57,24 @@ #include #include #include +#include /* Set this tunable to TRUE to replace corrupt data with 0x2f5baddb10c */ int zfs_send_corrupt_data = B_FALSE; int zfs_send_queue_length = 16 * 1024 * 1024; int zfs_recv_queue_length = 16 * 1024 * 1024; +/* Set this tunable to FALSE to disable setting of DRR_FLAG_FREERECORDS */ +int zfs_send_set_freerecords_bit = B_TRUE; static char *dmu_recv_tag = "dmu_recv_tag"; -static const char *recv_clone_name = "%recv"; +const char *recv_clone_name = "%recv"; #define BP_SPAN(datablkszsec, indblkshift, level) \ (((uint64_t)datablkszsec) << (SPA_MINBLOCKSHIFT + \ (level) * (indblkshift - SPA_BLKPTRSHIFT))) +static void byteswap_record(dmu_replay_record_t *drr); + struct send_thread_arg { bqueue_t q; dsl_dataset_t *ds; /* Dataset to traverse */ @@ -76,6 +82,7 @@ struct send_thread_arg { int flags; /* flags to pass to traverse_dataset */ int error_code; boolean_t cancel; + zbookmark_phys_t resume; }; struct send_block_record { @@ -98,8 +105,21 @@ dump_bytes_cb(void *arg) { dump_bytes_io_t *dbi = (dump_bytes_io_t *)arg; dmu_sendarg_t *dsp = dbi->dbi_dsp; - dsl_dataset_t *ds = dsp->dsa_os->os_dsl_dataset; + dsl_dataset_t *ds = dmu_objset_ds(dsp->dsa_os); ssize_t resid; /* have to get resid to get detailed errno */ + + /* + * The code does not rely on this (len being a multiple of 8). We keep + * this assertion because of the corresponding assertion in + * receive_read(). Keeping this assertion ensures that we do not + * inadvertently break backwards compatibility (causing the assertion + * in receive_read() to trigger on old software). + * + * Removing the assertions could be rolled into a new feature that uses + * data that isn't 8-byte aligned; if the assertions were removed, a + * feature flag would have to be added. + */ + ASSERT0(dbi->dbi_len % 8); dsp->dsa_err = vn_rdwr(UIO_WRITE, dsp->dsa_vp, @@ -168,6 +188,14 @@ dump_record(dmu_sendarg_t *dsp, void *payload, int payload_len) return (0); } +/* + * Fill in the drr_free struct, or perform aggregation if the previous record is + * also a free record, and the two are adjacent. + * + * Note that we send free records even for a full send, because we want to be + * able to receive a full send as a clone, which requires a list of all the free + * and freeobject records that were generated on the source. + */ static int dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset, uint64_t length) @@ -179,7 +207,7 @@ dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset, * that the receiving system doesn't have any dbufs in the range * being freed. This is always true because there is a one-record * constraint: we only send one WRITE record for any given - * object+offset. We know that the one-record constraint is + * object,offset. We know that the one-record constraint is * true because we always send data in increasing order by * object,offset. * @@ -191,15 +219,6 @@ dump_free(dmu_sendarg_t *dsp, uint64_t object, uint64_t offset, (object == dsp->dsa_last_data_object && offset > dsp->dsa_last_data_offset)); - /* - * If we are doing a non-incremental send, then there can't - * be any data in the dataset we're receiving into. Therefore - * a free record would simply be a no-op. Save space by not - * sending it to begin with. - */ - if (!dsp->dsa_incremental) - return (0); - if (length != -1ULL && offset + length < offset) length = -1ULL; @@ -377,10 +396,6 @@ dump_freeobjects(dmu_sendarg_t *dsp, uint64_t firstobj, uint64_t numobjs) { struct drr_freeobjects *drrfo = &(dsp->dsa_drr->drr_u.drr_freeobjects); - /* See comment in dump_free(). */ - if (!dsp->dsa_incremental) - return (0); - /* * If there is a pending op, but it's not PENDING_FREEOBJECTS, * push it out, since free block aggregation can only be done for @@ -427,6 +442,19 @@ dump_dnode(dmu_sendarg_t *dsp, uint64_t object, dnode_phys_t *dnp) { struct drr_object *drro = &(dsp->dsa_drr->drr_u.drr_object); + if (object < dsp->dsa_resume_object) { + /* + * Note: when resuming, we will visit all the dnodes in + * the block of dnodes that we are resuming from. In + * this case it's unnecessary to send the dnodes prior to + * the one we are resuming from. We should be at most one + * block's worth of dnodes behind the resume point. + */ + ASSERT3U(dsp->dsa_resume_object - object, <, + 1 << (DNODE_BLOCK_SHIFT - DNODE_SHIFT)); + return (0); + } + if (dnp == NULL || dnp->dn_type == DMU_OT_NONE) return (dump_freeobjects(dsp, object, 1)); @@ -444,6 +472,7 @@ dump_dnode(dmu_sendarg_t *dsp, uint64_t object, dnode_phys_t *dnp) drro->drr_bonustype = dnp->dn_bonustype; drro->drr_blksz = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT; drro->drr_bonuslen = dnp->dn_bonuslen; + drro->drr_dn_slots = dnp->dn_extra_slots + 1; drro->drr_checksumtype = dnp->dn_checksum; drro->drr_compress = dnp->dn_compress; drro->drr_toguid = dsp->dsa_toguid; @@ -507,6 +536,9 @@ send_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, uint64_t record_size; int err = 0; + ASSERT(zb->zb_object == DMU_META_DNODE_OBJECT || + zb->zb_object >= sta->resume.zb_object); + if (sta->cancel) return (SET_ERROR(EINTR)); @@ -543,8 +575,10 @@ send_traverse_thread(void *arg) struct send_block_record *data; if (st_arg->ds != NULL) { - err = traverse_dataset(st_arg->ds, st_arg->fromtxg, - st_arg->flags, send_cb, arg); + err = traverse_dataset_resume(st_arg->ds, + st_arg->fromtxg, &st_arg->resume, + st_arg->flags, send_cb, st_arg); + if (err != EINTR) st_arg->error_code = err; } @@ -569,11 +603,13 @@ do_dump(dmu_sendarg_t *dsa, struct send_block_record *data) spa_t *spa = ds->ds_dir->dd_pool->dp_spa; dmu_object_type_t type = bp ? BP_GET_TYPE(bp) : DMU_OT_NONE; int err = 0; - dnode_phys_t *blk; uint64_t dnobj; ASSERT3U(zb->zb_level, >=, 0); + ASSERT(zb->zb_object == DMU_META_DNODE_OBJECT || + zb->zb_object >= dsa->dsa_resume_object); + if (zb->zb_object != DMU_META_DNODE_OBJECT && DMU_OBJECT_IS_SPECIAL(zb->zb_object)) { return (0); @@ -589,7 +625,8 @@ do_dump(dmu_sendarg_t *dsa, struct send_block_record *data) } else if (zb->zb_level > 0 || type == DMU_OT_OBJSET) { return (0); } else if (type == DMU_OT_DNODE) { - int blksz = BP_GET_LSIZE(bp); + dnode_phys_t *blk; + int epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT; arc_flags_t aflags = ARC_FLAG_WAIT; arc_buf_t *abuf; int i; @@ -602,8 +639,8 @@ do_dump(dmu_sendarg_t *dsa, struct send_block_record *data) return (SET_ERROR(EIO)); blk = abuf->b_data; - dnobj = zb->zb_blkid * (blksz >> DNODE_SHIFT); - for (i = 0; i < blksz >> DNODE_SHIFT; i++) { + dnobj = zb->zb_blkid * epb; + for (i = 0; i < epb; i += blk[i].dn_extra_slots + 1) { err = dump_dnode(dsa, dnobj + i, blk + i); if (err != 0) break; @@ -635,6 +672,10 @@ do_dump(dmu_sendarg_t *dsa, struct send_block_record *data) uint64_t offset; ASSERT0(zb->zb_level); + ASSERT(zb->zb_object > dsa->dsa_resume_object || + (zb->zb_object == dsa->dsa_resume_object && + zb->zb_blkid * blksz >= dsa->dsa_resume_offset)); + if (arc_read(NULL, spa, bp, arc_getbuf_func, &abuf, ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &aflags, zb) != 0) { @@ -695,8 +736,10 @@ get_next_record(bqueue_t *bq, struct send_block_record *data) */ static int dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds, - zfs_bookmark_phys_t *ancestor_zb, boolean_t is_clone, boolean_t embedok, - boolean_t large_block_ok, int outfd, vnode_t *vp, offset_t *off) + zfs_bookmark_phys_t *ancestor_zb, + boolean_t is_clone, boolean_t embedok, boolean_t large_block_ok, int outfd, + uint64_t resumeobj, uint64_t resumeoff, + vnode_t *vp, offset_t *off) { objset_t *os; dmu_replay_record_t *drr; @@ -705,6 +748,8 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds, uint64_t fromtxg = 0; uint64_t featureflags = 0; struct send_thread_arg to_arg; + void *payload = NULL; + size_t payload_len = 0; struct send_block_record *to_data; err = dmu_objset_from_ds(to_ds, &os); @@ -719,6 +764,8 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds, DMU_SET_STREAM_HDRTYPE(drr->drr_u.drr_begin.drr_versioninfo, DMU_SUBSTREAM); + bzero(&to_arg, sizeof (to_arg)); + #ifdef _KERNEL if (dmu_objset_type(os) == DMU_OST_ZFS) { uint64_t version; @@ -735,6 +782,8 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds, if (large_block_ok && to_ds->ds_feature_inuse[SPA_FEATURE_LARGE_BLOCKS]) featureflags |= DMU_BACKUP_FEATURE_LARGE_BLOCKS; + if (to_ds->ds_feature_inuse[SPA_FEATURE_LARGE_DNODE]) + featureflags |= DMU_BACKUP_FEATURE_LARGE_DNODE; if (embedok && spa_feature_is_active(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA)) { featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA; @@ -742,6 +791,10 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds, featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA_LZ4; } + if (resumeobj != 0 || resumeoff != 0) { + featureflags |= DMU_BACKUP_FEATURE_RESUMING; + } + DMU_SET_FEATUREFLAGS(drr->drr_u.drr_begin.drr_versioninfo, featureflags); @@ -753,6 +806,8 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds, drr->drr_u.drr_begin.drr_toguid = dsl_dataset_phys(to_ds)->ds_guid; if (dsl_dataset_phys(to_ds)->ds_flags & DS_FLAG_CI_DATASET) drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_CI_DATA; + if (zfs_send_set_freerecords_bit) + drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_FREERECORDS; if (ancestor_zb != NULL) { drr->drr_u.drr_begin.drr_fromguid = @@ -775,8 +830,9 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds, dsp->dsa_off = off; dsp->dsa_toguid = dsl_dataset_phys(to_ds)->ds_guid; dsp->dsa_pending_op = PENDING_NONE; - dsp->dsa_incremental = (ancestor_zb != NULL); dsp->dsa_featureflags = featureflags; + dsp->dsa_resume_object = resumeobj; + dsp->dsa_resume_offset = resumeoff; mutex_enter(&to_ds->ds_sendstream_lock); list_insert_head(&to_ds->ds_sendstreams, dsp); @@ -785,7 +841,26 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *to_ds, dsl_dataset_long_hold(to_ds, FTAG); dsl_pool_rele(dp, tag); - if (dump_record(dsp, NULL, 0) != 0) { + if (resumeobj != 0 || resumeoff != 0) { + dmu_object_info_t to_doi; + nvlist_t *nvl; + err = dmu_object_info(os, resumeobj, &to_doi); + if (err != 0) + goto out; + SET_BOOKMARK(&to_arg.resume, to_ds->ds_object, resumeobj, 0, + resumeoff / to_doi.doi_data_block_size); + + nvl = fnvlist_alloc(); + fnvlist_add_uint64(nvl, "resume_object", resumeobj); + fnvlist_add_uint64(nvl, "resume_offset", resumeoff); + payload = fnvlist_pack(nvl, &payload_len); + drr->drr_payloadlen = payload_len; + fnvlist_free(nvl); + } + + err = dump_record(dsp, payload, payload_len); + fnvlist_pack_free(payload, payload_len); + if (err != 0) { err = dsp->dsa_err; goto out; } @@ -895,19 +970,19 @@ dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap, is_clone = (fromds->ds_dir != ds->ds_dir); dsl_dataset_rele(fromds, FTAG); err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone, - embedok, large_block_ok, outfd, vp, off); + embedok, large_block_ok, outfd, 0, 0, vp, off); } else { err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE, - embedok, large_block_ok, outfd, vp, off); + embedok, large_block_ok, outfd, 0, 0, vp, off); } dsl_dataset_rele(ds, FTAG); return (err); } int -dmu_send(const char *tosnap, const char *fromsnap, - boolean_t embedok, boolean_t large_block_ok, - int outfd, vnode_t *vp, offset_t *off) +dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok, + boolean_t large_block_ok, int outfd, uint64_t resumeobj, uint64_t resumeoff, + vnode_t *vp, offset_t *off) { dsl_pool_t *dp; dsl_dataset_t *ds; @@ -974,10 +1049,12 @@ dmu_send(const char *tosnap, const char *fromsnap, return (err); } err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone, - embedok, large_block_ok, outfd, vp, off); + embedok, large_block_ok, + outfd, resumeobj, resumeoff, vp, off); } else { err = dmu_send_impl(FTAG, dp, ds, NULL, B_FALSE, - embedok, large_block_ok, outfd, vp, off); + embedok, large_block_ok, + outfd, resumeobj, resumeoff, vp, off); } if (owned) dsl_dataset_disown(ds, FTAG); @@ -1217,6 +1294,7 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx) /* already checked */ ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC); + ASSERT(!(featureflags & DMU_BACKUP_FEATURE_RESUMING)); if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == DMU_COMPOUNDSTREAM || @@ -1229,6 +1307,10 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx) spa_version(dp->dp_spa) < SPA_VERSION_SA) return (SET_ERROR(ENOTSUP)); + if (drba->drba_cookie->drc_resumable && + !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EXTENSIBLE_DATASET)) + return (SET_ERROR(ENOTSUP)); + /* * The receiving code doesn't know how to translate a WRITE_EMBEDDED * record to a plan WRITE record, so the pool must have the @@ -1251,12 +1333,21 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx) !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LARGE_BLOCKS)) return (SET_ERROR(ENOTSUP)); + /* + * The receiving code doesn't know how to translate large dnodes + * to smaller ones, so the pool must have the LARGE_DNODE + * feature enabled if the stream has LARGE_DNODE. + */ + if ((featureflags & DMU_BACKUP_FEATURE_LARGE_DNODE) && + !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LARGE_DNODE)) + return (SET_ERROR(ENOTSUP)); + error = dsl_dataset_hold(dp, tofs, FTAG, &ds); if (error == 0) { /* target fs already exists; recv into temp clone */ /* Can't recv a clone into an existing fs */ - if (flags & DRR_FLAG_CLONE) { + if (flags & DRR_FLAG_CLONE || drba->drba_origin) { dsl_dataset_rele(ds, FTAG); return (SET_ERROR(EINVAL)); } @@ -1265,7 +1356,7 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx) dsl_dataset_rele(ds, FTAG); } else if (error == ENOENT) { /* target fs does not exist; must be a full backup or clone */ - char buf[MAXNAMELEN]; + char buf[ZFS_MAX_DATASET_NAME_LEN]; /* * If it's a non-clone incremental, we are missing the @@ -1275,8 +1366,17 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx) drba->drba_origin)) return (SET_ERROR(ENOENT)); + /* + * If we're receiving a full send as a clone, and it doesn't + * contain all the necessary free records and freeobject + * records, reject it. + */ + if (fromguid == 0 && drba->drba_origin && + !(flags & DRR_FLAG_FREERECORDS)) + return (SET_ERROR(EINVAL)); + /* Open the parent of tofs */ - ASSERT3U(strlen(tofs), <, MAXNAMELEN); + ASSERT3U(strlen(tofs), <, sizeof (buf)); (void) strlcpy(buf, tofs, strrchr(tofs, '/') - tofs + 1); error = dsl_dataset_hold(dp, buf, FTAG, &ds); if (error != 0) @@ -1314,7 +1414,8 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx) dsl_dataset_rele(ds, FTAG); return (SET_ERROR(EINVAL)); } - if (dsl_dataset_phys(origin)->ds_guid != fromguid) { + if (dsl_dataset_phys(origin)->ds_guid != fromguid && + fromguid != 0) { dsl_dataset_rele(origin, FTAG); dsl_dataset_rele(ds, FTAG); return (SET_ERROR(ENODEV)); @@ -1332,15 +1433,16 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx) { dmu_recv_begin_arg_t *drba = arg; dsl_pool_t *dp = dmu_tx_pool(tx); + objset_t *mos = dp->dp_meta_objset; struct drr_begin *drrb = drba->drba_cookie->drc_drrb; const char *tofs = drba->drba_cookie->drc_tofs; dsl_dataset_t *ds, *newds; uint64_t dsobj; int error; - uint64_t crflags; + uint64_t crflags = 0; - crflags = (drrb->drr_flags & DRR_FLAG_CI_DATA) ? - DS_FLAG_CI_DATASET : 0; + if (drrb->drr_flags & DRR_FLAG_CI_DATA) + crflags |= DS_FLAG_CI_DATASET; error = dsl_dataset_hold(dp, tofs, FTAG, &ds); if (error == 0) { @@ -1378,6 +1480,32 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx) } VERIFY0(dsl_dataset_own_obj(dp, dsobj, dmu_recv_tag, &newds)); + if (drba->drba_cookie->drc_resumable) { + uint64_t one = 1; + uint64_t zero = 0; + + dsl_dataset_zapify(newds, tx); + if (drrb->drr_fromguid != 0) { + VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_FROMGUID, + 8, 1, &drrb->drr_fromguid, tx)); + } + VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TOGUID, + 8, 1, &drrb->drr_toguid, tx)); + VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TONAME, + 1, strlen(drrb->drr_toname) + 1, drrb->drr_toname, tx)); + VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OBJECT, + 8, 1, &one, tx)); + VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OFFSET, + 8, 1, &zero, tx)); + VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_BYTES, + 8, 1, &zero, tx)); + if (DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo) & + DMU_BACKUP_FEATURE_EMBED_DATA) { + VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_EMBEDOK, + 8, 1, &one, tx)); + } + } + dmu_buf_will_dirty(newds->ds_dbuf, tx); dsl_dataset_phys(newds)->ds_flags |= DS_FLAG_INCONSISTENT; @@ -1395,56 +1523,194 @@ dmu_recv_begin_sync(void *arg, dmu_tx_t *tx) spa_history_log_internal_ds(newds, "receive", tx, ""); } +static int +dmu_recv_resume_begin_check(void *arg, dmu_tx_t *tx) +{ + dmu_recv_begin_arg_t *drba = arg; + dsl_pool_t *dp = dmu_tx_pool(tx); + struct drr_begin *drrb = drba->drba_cookie->drc_drrb; + int error; + uint64_t featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo); + dsl_dataset_t *ds; + const char *tofs = drba->drba_cookie->drc_tofs; + uint64_t val; + + /* 6 extra bytes for /%recv */ + char recvname[ZFS_MAX_DATASET_NAME_LEN + 6]; + + /* already checked */ + ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC); + ASSERT(featureflags & DMU_BACKUP_FEATURE_RESUMING); + + if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) == + DMU_COMPOUNDSTREAM || + drrb->drr_type >= DMU_OST_NUMTYPES) + return (SET_ERROR(EINVAL)); + + /* Verify pool version supports SA if SA_SPILL feature set */ + if ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) && + spa_version(dp->dp_spa) < SPA_VERSION_SA) + return (SET_ERROR(ENOTSUP)); + + /* + * The receiving code doesn't know how to translate a WRITE_EMBEDDED + * record to a plain WRITE record, so the pool must have the + * EMBEDDED_DATA feature enabled if the stream has WRITE_EMBEDDED + * records. Same with WRITE_EMBEDDED records that use LZ4 compression. + */ + if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) && + !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA)) + return (SET_ERROR(ENOTSUP)); + if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA_LZ4) && + !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS)) + return (SET_ERROR(ENOTSUP)); + + (void) snprintf(recvname, sizeof (recvname), "%s/%s", + tofs, recv_clone_name); + + if (dsl_dataset_hold(dp, recvname, FTAG, &ds) != 0) { + /* %recv does not exist; continue in tofs */ + error = dsl_dataset_hold(dp, tofs, FTAG, &ds); + if (error != 0) + return (error); + } + + /* check that ds is marked inconsistent */ + if (!DS_IS_INCONSISTENT(ds)) { + dsl_dataset_rele(ds, FTAG); + return (SET_ERROR(EINVAL)); + } + + /* check that there is resuming data, and that the toguid matches */ + if (!dsl_dataset_is_zapified(ds)) { + dsl_dataset_rele(ds, FTAG); + return (SET_ERROR(EINVAL)); + } + error = zap_lookup(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val); + if (error != 0 || drrb->drr_toguid != val) { + dsl_dataset_rele(ds, FTAG); + return (SET_ERROR(EINVAL)); + } + + /* + * Check if the receive is still running. If so, it will be owned. + * Note that nothing else can own the dataset (e.g. after the receive + * fails) because it will be marked inconsistent. + */ + if (dsl_dataset_has_owner(ds)) { + dsl_dataset_rele(ds, FTAG); + return (SET_ERROR(EBUSY)); + } + + /* There should not be any snapshots of this fs yet. */ + if (ds->ds_prev != NULL && ds->ds_prev->ds_dir == ds->ds_dir) { + dsl_dataset_rele(ds, FTAG); + return (SET_ERROR(EINVAL)); + } + + /* + * Note: resume point will be checked when we process the first WRITE + * record. + */ + + /* check that the origin matches */ + val = 0; + (void) zap_lookup(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val); + if (drrb->drr_fromguid != val) { + dsl_dataset_rele(ds, FTAG); + return (SET_ERROR(EINVAL)); + } + + dsl_dataset_rele(ds, FTAG); + return (0); +} + +static void +dmu_recv_resume_begin_sync(void *arg, dmu_tx_t *tx) +{ + dmu_recv_begin_arg_t *drba = arg; + dsl_pool_t *dp = dmu_tx_pool(tx); + const char *tofs = drba->drba_cookie->drc_tofs; + dsl_dataset_t *ds; + uint64_t dsobj; + /* 6 extra bytes for /%recv */ + char recvname[ZFS_MAX_DATASET_NAME_LEN + 6]; + + (void) snprintf(recvname, sizeof (recvname), "%s/%s", + tofs, recv_clone_name); + + if (dsl_dataset_hold(dp, recvname, FTAG, &ds) != 0) { + /* %recv does not exist; continue in tofs */ + VERIFY0(dsl_dataset_hold(dp, tofs, FTAG, &ds)); + drba->drba_cookie->drc_newfs = B_TRUE; + } + + /* clear the inconsistent flag so that we can own it */ + ASSERT(DS_IS_INCONSISTENT(ds)); + dmu_buf_will_dirty(ds->ds_dbuf, tx); + dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT; + dsobj = ds->ds_object; + dsl_dataset_rele(ds, FTAG); + + VERIFY0(dsl_dataset_own_obj(dp, dsobj, dmu_recv_tag, &ds)); + + dmu_buf_will_dirty(ds->ds_dbuf, tx); + dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_INCONSISTENT; + + ASSERT(!BP_IS_HOLE(dsl_dataset_get_blkptr(ds))); + + drba->drba_cookie->drc_ds = ds; + + spa_history_log_internal_ds(ds, "resume receive", tx, ""); +} + /* * NB: callers *MUST* call dmu_recv_stream() if dmu_recv_begin() * succeeds; otherwise we will leak the holds on the datasets. */ int -dmu_recv_begin(char *tofs, char *tosnap, struct drr_begin *drrb, - boolean_t force, char *origin, dmu_recv_cookie_t *drc) +dmu_recv_begin(char *tofs, char *tosnap, dmu_replay_record_t *drr_begin, + boolean_t force, boolean_t resumable, char *origin, dmu_recv_cookie_t *drc) { dmu_recv_begin_arg_t drba = { 0 }; - dmu_replay_record_t *drr; bzero(drc, sizeof (dmu_recv_cookie_t)); - drc->drc_drrb = drrb; + drc->drc_drr_begin = drr_begin; + drc->drc_drrb = &drr_begin->drr_u.drr_begin; drc->drc_tosnap = tosnap; drc->drc_tofs = tofs; drc->drc_force = force; + drc->drc_resumable = resumable; drc->drc_cred = CRED(); - if (drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) + if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) { drc->drc_byteswap = B_TRUE; - else if (drrb->drr_magic != DMU_BACKUP_MAGIC) - return (SET_ERROR(EINVAL)); - - drr = kmem_zalloc(sizeof (dmu_replay_record_t), KM_SLEEP); - drr->drr_type = DRR_BEGIN; - drr->drr_u.drr_begin = *drc->drc_drrb; - if (drc->drc_byteswap) { - fletcher_4_incremental_byteswap(drr, + fletcher_4_incremental_byteswap(drr_begin, sizeof (dmu_replay_record_t), &drc->drc_cksum); - } else { - fletcher_4_incremental_native(drr, + byteswap_record(drr_begin); + } else if (drc->drc_drrb->drr_magic == DMU_BACKUP_MAGIC) { + fletcher_4_incremental_native(drr_begin, sizeof (dmu_replay_record_t), &drc->drc_cksum); - } - kmem_free(drr, sizeof (dmu_replay_record_t)); - - if (drc->drc_byteswap) { - drrb->drr_magic = BSWAP_64(drrb->drr_magic); - drrb->drr_versioninfo = BSWAP_64(drrb->drr_versioninfo); - drrb->drr_creation_time = BSWAP_64(drrb->drr_creation_time); - drrb->drr_type = BSWAP_32(drrb->drr_type); - drrb->drr_toguid = BSWAP_64(drrb->drr_toguid); - drrb->drr_fromguid = BSWAP_64(drrb->drr_fromguid); + } else { + return (SET_ERROR(EINVAL)); } drba.drba_origin = origin; drba.drba_cookie = drc; drba.drba_cred = CRED(); - return (dsl_sync_task(tofs, dmu_recv_begin_check, dmu_recv_begin_sync, - &drba, 5, ZFS_SPACE_CHECK_NORMAL)); + if (DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo) & + DMU_BACKUP_FEATURE_RESUMING) { + return (dsl_sync_task(tofs, + dmu_recv_resume_begin_check, dmu_recv_resume_begin_sync, + &drba, 5, ZFS_SPACE_CHECK_NORMAL)); + } else { + return (dsl_sync_task(tofs, + dmu_recv_begin_check, dmu_recv_begin_sync, + &drba, 5, ZFS_SPACE_CHECK_NORMAL)); + } } struct receive_record_arg { @@ -1456,6 +1722,7 @@ struct receive_record_arg { */ arc_buf_t *write_buf; int payload_size; + uint64_t bytes_read; /* bytes read from stream when record created */ boolean_t eos_marker; /* Marks the end of the stream */ bqueue_node_t node; }; @@ -1464,6 +1731,7 @@ struct receive_writer_arg { objset_t *os; boolean_t byteswap; bqueue_t q; + /* * These three args are used to signal to the main thread that we're * done. @@ -1471,15 +1739,34 @@ struct receive_writer_arg { kmutex_t mutex; kcondvar_t cv; boolean_t done; + int err; /* A map from guid to dataset to help handle dedup'd streams. */ avl_tree_t *guid_to_ds_map; + boolean_t resumable; + uint64_t last_object, last_offset; + uint64_t bytes_read; /* bytes read when current record created */ +}; + +struct objlist { + list_t list; /* List of struct receive_objnode. */ + /* + * Last object looked up. Used to assert that objects are being looked + * up in ascending order. + */ + uint64_t last_lookup; +}; + +struct receive_objnode { + list_node_t node; + uint64_t object; }; struct receive_arg { objset_t *os; vnode_t *vp; /* The vnode to read the stream from */ uint64_t voff; /* The current offset in the stream */ + uint64_t bytes_read; /* * A record that has had its payload read in, but hasn't yet been handed * off to the worker thread. @@ -1492,12 +1779,7 @@ struct receive_arg { int err; boolean_t byteswap; /* Sorted list of objects not to issue prefetches for. */ - list_t ignore_obj_list; -}; - -struct receive_ign_obj_node { - list_node_t node; - uint64_t object; + struct objlist ignore_objlist; }; typedef struct guid_map_entry { @@ -1540,7 +1822,10 @@ receive_read(struct receive_arg *ra, int len, void *buf) { int done = 0; - /* some things will require 8-byte alignment, so everything must */ + /* + * The code doesn't rely on this (lengths being multiples of 8). See + * comment in dump_bytes. + */ ASSERT0(len % 8); while (done < len) { @@ -1551,14 +1836,21 @@ receive_read(struct receive_arg *ra, int len, void *buf) ra->voff, UIO_SYSSPACE, FAPPEND, RLIM64_INFINITY, CRED(), &resid); - if (resid == len - done) - ra->err = SET_ERROR(EINVAL); + if (resid == len - done) { + /* + * Note: ECKSUM indicates that the receive + * was interrupted and can potentially be resumed. + */ + ra->err = SET_ERROR(ECKSUM); + } ra->voff += len - done - resid; done = len - resid; if (ra->err != 0) return (ra->err); } + ra->bytes_read += len; + ASSERT3U(done, ==, len); return (0); } @@ -1657,10 +1949,48 @@ deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size) return (1); } else { return (1 + - ((DN_MAX_BONUSLEN - bonus_size) >> SPA_BLKPTRSHIFT)); + ((DN_OLD_MAX_BONUSLEN - + MIN(DN_OLD_MAX_BONUSLEN, bonus_size)) >> SPA_BLKPTRSHIFT)); } } +static void +save_resume_state(struct receive_writer_arg *rwa, + uint64_t object, uint64_t offset, dmu_tx_t *tx) +{ + int txgoff = dmu_tx_get_txg(tx) & TXG_MASK; + + if (!rwa->resumable) + return; + + /* + * We use ds_resume_bytes[] != 0 to indicate that we need to + * update this on disk, so it must not be 0. + */ + ASSERT(rwa->bytes_read != 0); + + /* + * We only resume from write records, which have a valid + * (non-meta-dnode) object number. + */ + ASSERT(object != 0); + + /* + * For resuming to work correctly, we must receive records in order, + * sorted by object,offset. This is checked by the callers, but + * assert it here for good measure. + */ + ASSERT3U(object, >=, rwa->os->os_dsl_dataset->ds_resume_object[txgoff]); + ASSERT(object != rwa->os->os_dsl_dataset->ds_resume_object[txgoff] || + offset >= rwa->os->os_dsl_dataset->ds_resume_offset[txgoff]); + ASSERT3U(rwa->bytes_read, >=, + rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff]); + + rwa->os->os_dsl_dataset->ds_resume_object[txgoff] = object; + rwa->os->os_dsl_dataset->ds_resume_offset[txgoff] = offset; + rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff] = rwa->bytes_read; +} + noinline static int receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, void *data) @@ -1678,7 +2008,8 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) || drro->drr_blksz < SPA_MINBLOCKSIZE || drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(rwa->os)) || - drro->drr_bonuslen > DN_MAX_BONUSLEN) { + drro->drr_bonuslen > + DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(rwa->os)))) { return (SET_ERROR(EINVAL)); } @@ -1718,9 +2049,10 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, if (object == DMU_NEW_OBJECT) { /* currently free, want to be allocated */ - err = dmu_object_claim(rwa->os, drro->drr_object, + err = dmu_object_claim_dnsize(rwa->os, drro->drr_object, drro->drr_type, drro->drr_blksz, - drro->drr_bonustype, drro->drr_bonuslen, tx); + drro->drr_bonustype, drro->drr_bonuslen, + drro->drr_dn_slots << DNODE_SHIFT, tx); } else if (drro->drr_type != doi.doi_type || drro->drr_blksz != doi.doi_data_block_size || drro->drr_bonustype != doi.doi_bonus_type || @@ -1757,6 +2089,7 @@ receive_object(struct receive_writer_arg *rwa, struct drr_object *drro, dmu_buf_rele(db, FTAG); } dmu_tx_commit(tx); + return (0); } @@ -1766,22 +2099,31 @@ receive_freeobjects(struct receive_writer_arg *rwa, struct drr_freeobjects *drrfo) { uint64_t obj; + int next_err = 0; if (drrfo->drr_firstobj + drrfo->drr_numobjs < drrfo->drr_firstobj) return (SET_ERROR(EINVAL)); - for (obj = drrfo->drr_firstobj; - obj < drrfo->drr_firstobj + drrfo->drr_numobjs; - (void) dmu_object_next(rwa->os, &obj, FALSE, 0)) { + for (obj = drrfo->drr_firstobj == 0 ? 1 : drrfo->drr_firstobj; + obj < drrfo->drr_firstobj + drrfo->drr_numobjs && next_err == 0; + next_err = dmu_object_next(rwa->os, &obj, FALSE, 0)) { + dmu_object_info_t doi; int err; - if (dmu_object_info(rwa->os, obj, NULL) != 0) + err = dmu_object_info(rwa->os, obj, &doi); + if (err == ENOENT) { + obj++; continue; + } else if (err != 0) { + return (err); + } err = dmu_free_long_object(rwa->os, obj); if (err != 0) return (err); } + if (next_err != ESRCH) + return (next_err); return (0); } @@ -1797,6 +2139,18 @@ receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw, !DMU_OT_IS_VALID(drrw->drr_type)) return (SET_ERROR(EINVAL)); + /* + * For resuming to work, records must be in increasing order + * by (object, offset). + */ + if (drrw->drr_object < rwa->last_object || + (drrw->drr_object == rwa->last_object && + drrw->drr_offset < rwa->last_offset)) { + return (SET_ERROR(EINVAL)); + } + rwa->last_object = drrw->drr_object; + rwa->last_offset = drrw->drr_offset; + if (dmu_object_info(rwa->os, drrw->drr_object, NULL) != 0) return (SET_ERROR(EINVAL)); @@ -1819,8 +2173,17 @@ receive_write(struct receive_writer_arg *rwa, struct drr_write *drrw, if (dmu_bonus_hold(rwa->os, drrw->drr_object, FTAG, &bonus) != 0) return (SET_ERROR(EINVAL)); dmu_assign_arcbuf(bonus, drrw->drr_offset, abuf, tx); + + /* + * Note: If the receive fails, we want the resume stream to start + * with the same record that we last successfully received (as opposed + * to the next record), so that we can verify that we are + * resuming from the correct location. + */ + save_resume_state(rwa, drrw->drr_object, drrw->drr_offset, tx); dmu_tx_commit(tx); dmu_buf_rele(bonus, FTAG); + return (0); } @@ -1879,43 +2242,48 @@ receive_write_byref(struct receive_writer_arg *rwa, dmu_write(rwa->os, drrwbr->drr_object, drrwbr->drr_offset, drrwbr->drr_length, dbp->db_data, tx); dmu_buf_rele(dbp, FTAG); + + /* See comment in restore_write. */ + save_resume_state(rwa, drrwbr->drr_object, drrwbr->drr_offset, tx); dmu_tx_commit(tx); return (0); } static int receive_write_embedded(struct receive_writer_arg *rwa, - struct drr_write_embedded *drrwnp, void *data) + struct drr_write_embedded *drrwe, void *data) { dmu_tx_t *tx; int err; - if (drrwnp->drr_offset + drrwnp->drr_length < drrwnp->drr_offset) + if (drrwe->drr_offset + drrwe->drr_length < drrwe->drr_offset) return (EINVAL); - if (drrwnp->drr_psize > BPE_PAYLOAD_SIZE) + if (drrwe->drr_psize > BPE_PAYLOAD_SIZE) return (EINVAL); - if (drrwnp->drr_etype >= NUM_BP_EMBEDDED_TYPES) + if (drrwe->drr_etype >= NUM_BP_EMBEDDED_TYPES) return (EINVAL); - if (drrwnp->drr_compression >= ZIO_COMPRESS_FUNCTIONS) + if (drrwe->drr_compression >= ZIO_COMPRESS_FUNCTIONS) return (EINVAL); tx = dmu_tx_create(rwa->os); - dmu_tx_hold_write(tx, drrwnp->drr_object, - drrwnp->drr_offset, drrwnp->drr_length); + dmu_tx_hold_write(tx, drrwe->drr_object, + drrwe->drr_offset, drrwe->drr_length); err = dmu_tx_assign(tx, TXG_WAIT); if (err != 0) { dmu_tx_abort(tx); return (err); } - dmu_write_embedded(rwa->os, drrwnp->drr_object, - drrwnp->drr_offset, data, drrwnp->drr_etype, - drrwnp->drr_compression, drrwnp->drr_lsize, drrwnp->drr_psize, + dmu_write_embedded(rwa->os, drrwe->drr_object, + drrwe->drr_offset, data, drrwe->drr_etype, + drrwe->drr_compression, drrwe->drr_lsize, drrwe->drr_psize, rwa->byteswap ^ ZFS_HOST_BYTEORDER, tx); + /* See comment in restore_write. */ + save_resume_state(rwa, drrwe->drr_object, drrwe->drr_offset, tx); dmu_tx_commit(tx); return (0); } @@ -1989,10 +2357,16 @@ receive_free(struct receive_writer_arg *rwa, struct drr_free *drrf) static void dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc) { - char name[MAXNAMELEN]; - dsl_dataset_name(drc->drc_ds, name); - dsl_dataset_disown(drc->drc_ds, dmu_recv_tag); - (void) dsl_destroy_head(name); + if (drc->drc_resumable) { + /* wait for our resume state to be written to disk */ + txg_wait_synced(drc->drc_ds->ds_dir->dd_pool, 0); + dsl_dataset_disown(drc->drc_ds, dmu_recv_tag); + } else { + char name[ZFS_MAX_DATASET_NAME_LEN]; + dsl_dataset_name(drc->drc_ds, name); + dsl_dataset_disown(drc->drc_ds, dmu_recv_tag); + (void) dsl_destroy_head(name); + } } static void @@ -2021,12 +2395,17 @@ receive_read_payload_and_next_header(struct receive_arg *ra, int len, void *buf) if (len != 0) { ASSERT3U(len, <=, SPA_MAXBLOCKSIZE); - ra->rrd->payload = buf; - ra->rrd->payload_size = len; - err = receive_read(ra, len, ra->rrd->payload); + err = receive_read(ra, len, buf); if (err != 0) return (err); - receive_cksum(ra, len, ra->rrd->payload); + receive_cksum(ra, len, buf); + + /* note: rrd is NULL when reading the begin record's payload */ + if (ra->rrd != NULL) { + ra->rrd->payload = buf; + ra->rrd->payload_size = len; + ra->rrd->bytes_read = ra->bytes_read; + } } ra->prev_cksum = ra->cksum; @@ -2034,6 +2413,7 @@ receive_read_payload_and_next_header(struct receive_arg *ra, int len, void *buf) ra->next_rrd = kmem_zalloc(sizeof (*ra->next_rrd), KM_SLEEP); err = receive_read(ra, sizeof (ra->next_rrd->header), &ra->next_rrd->header); + ra->next_rrd->bytes_read = ra->bytes_read; if (err != 0) { kmem_free(ra->next_rrd, sizeof (*ra->next_rrd)); ra->next_rrd = NULL; @@ -2073,6 +2453,70 @@ receive_read_payload_and_next_header(struct receive_arg *ra, int len, void *buf) return (0); } +static void +objlist_create(struct objlist *list) +{ + list_create(&list->list, sizeof (struct receive_objnode), + offsetof(struct receive_objnode, node)); + list->last_lookup = 0; +} + +static void +objlist_destroy(struct objlist *list) +{ + struct receive_objnode *n; + + for (n = list_remove_head(&list->list); + n != NULL; n = list_remove_head(&list->list)) { + kmem_free(n, sizeof (*n)); + } + list_destroy(&list->list); +} + +/* + * This function looks through the objlist to see if the specified object number + * is contained in the objlist. In the process, it will remove all object + * numbers in the list that are smaller than the specified object number. Thus, + * any lookup of an object number smaller than a previously looked up object + * number will always return false; therefore, all lookups should be done in + * ascending order. + */ +static boolean_t +objlist_exists(struct objlist *list, uint64_t object) +{ + struct receive_objnode *node = list_head(&list->list); + ASSERT3U(object, >=, list->last_lookup); + list->last_lookup = object; + while (node != NULL && node->object < object) { + VERIFY3P(node, ==, list_remove_head(&list->list)); + kmem_free(node, sizeof (*node)); + node = list_head(&list->list); + } + return (node != NULL && node->object == object); +} + +/* + * The objlist is a list of object numbers stored in ascending order. However, + * the insertion of new object numbers does not seek out the correct location to + * store a new object number; instead, it appends it to the list for simplicity. + * Thus, any users must take care to only insert new object numbers in ascending + * order. + */ +static void +objlist_insert(struct objlist *list, uint64_t object) +{ + struct receive_objnode *node = kmem_zalloc(sizeof (*node), KM_SLEEP); + node->object = object; +#ifdef ZFS_DEBUG + { + struct receive_objnode *last_object = list_tail(&list->list); + uint64_t last_objnum = (last_object != NULL ? last_object->object : 0); + ASSERT3U(node->object, >, last_objnum); + } +#endif + list_insert_tail(&list->list, node); +} + /* * Issue the prefetch reads for any necessary indirect blocks. * @@ -2095,13 +2539,7 @@ static void receive_read_prefetch(struct receive_arg *ra, uint64_t object, uint64_t offset, uint64_t length) { - struct receive_ign_obj_node *node = list_head(&ra->ignore_obj_list); - while (node != NULL && node->object < object) { - VERIFY3P(node, ==, list_remove_head(&ra->ignore_obj_list)); - kmem_free(node, sizeof (*node)); - node = list_head(&ra->ignore_obj_list); - } - if (node == NULL || node->object > object) { + if (!objlist_exists(&ra->ignore_objlist, object)) { dmu_prefetch(ra->os, object, 1, offset, length, ZIO_PRIORITY_SYNC_READ); } @@ -2134,20 +2572,7 @@ receive_read_record(struct receive_arg *ra) */ if (err == ENOENT || (err == 0 && doi.doi_data_block_size != drro->drr_blksz)) { - struct receive_ign_obj_node *node = - kmem_zalloc(sizeof (*node), - KM_SLEEP); - node->object = drro->drr_object; -#ifdef ZFS_DEBUG - { - struct receive_ign_obj_node *last_object = - list_tail(&ra->ignore_obj_list); - uint64_t last_objnum = (last_object != NULL ? - last_object->object : 0); - ASSERT3U(node->object, >, last_objnum); - } -#endif - list_insert_tail(&ra->ignore_obj_list, node); + objlist_insert(&ra->ignore_objlist, drro->drr_object); err = 0; } return (err); @@ -2213,7 +2638,7 @@ receive_read_record(struct receive_arg *ra) { struct drr_end *drre = &ra->rrd->header.drr_u.drr_end; if (!ZIO_CHECKSUM_EQUAL(ra->prev_cksum, drre->drr_checksum)) - return (SET_ERROR(EINVAL)); + return (SET_ERROR(ECKSUM)); return (0); } case DRR_SPILL: @@ -2240,6 +2665,10 @@ receive_process_record(struct receive_writer_arg *rwa, { int err; + /* Processing in order, therefore bytes_read should be increasing. */ + ASSERT3U(rrd->bytes_read, >=, rwa->bytes_read); + rwa->bytes_read = rrd->bytes_read; + switch (rrd->header.drr_type) { case DRR_OBJECT: { @@ -2334,6 +2763,32 @@ receive_writer_thread(void *arg) mutex_exit(&rwa->mutex); } +static int +resume_check(struct receive_arg *ra, nvlist_t *begin_nvl) +{ + uint64_t val; + objset_t *mos = dmu_objset_pool(ra->os)->dp_meta_objset; + uint64_t dsobj = dmu_objset_id(ra->os); + uint64_t resume_obj, resume_off; + + if (nvlist_lookup_uint64(begin_nvl, + "resume_object", &resume_obj) != 0 || + nvlist_lookup_uint64(begin_nvl, + "resume_offset", &resume_off) != 0) { + return (SET_ERROR(EINVAL)); + } + VERIFY0(zap_lookup(mos, dsobj, + DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val)); + if (resume_obj != val) + return (SET_ERROR(EINVAL)); + VERIFY0(zap_lookup(mos, dsobj, + DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val)); + if (resume_off != val) + return (SET_ERROR(EINVAL)); + + return (0); +} + /* * Read in the stream's records, one by one, and apply them to the pool. There * are two threads involved; the thread that calls this function will spin up a @@ -2354,7 +2809,9 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp, struct receive_arg *ra; struct receive_writer_arg *rwa; int featureflags; - struct receive_ign_obj_node *n; + uint32_t payloadlen; + void *payload; + nvlist_t *begin_nvl = NULL; ra = kmem_zalloc(sizeof (*ra), KM_SLEEP); rwa = kmem_zalloc(sizeof (*rwa), KM_SLEEP); @@ -2363,8 +2820,14 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp, ra->cksum = drc->drc_cksum; ra->vp = vp; ra->voff = *voffp; - list_create(&ra->ignore_obj_list, sizeof (struct receive_ign_obj_node), - offsetof(struct receive_ign_obj_node, node)); + + if (dsl_dataset_is_zapified(drc->drc_ds)) { + (void) zap_lookup(drc->drc_ds->ds_dir->dd_pool->dp_meta_objset, + drc->drc_ds->ds_object, DS_FIELD_RESUME_BYTES, + sizeof (ra->bytes_read), 1, &ra->bytes_read); + } + + objlist_create(&ra->ignore_objlist); /* these were verified in dmu_recv_begin */ ASSERT3U(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo), ==, @@ -2415,9 +2878,29 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp, drc->drc_guid_to_ds_map = rwa->guid_to_ds_map; } - err = receive_read_payload_and_next_header(ra, 0, NULL); - if (err) + payloadlen = drc->drc_drr_begin->drr_payloadlen; + payload = NULL; + if (payloadlen != 0) + payload = kmem_alloc(payloadlen, KM_SLEEP); + + err = receive_read_payload_and_next_header(ra, payloadlen, payload); + if (err != 0) { + if (payloadlen != 0) + kmem_free(payload, payloadlen); goto out; + } + if (payloadlen != 0) { + err = nvlist_unpack(payload, payloadlen, &begin_nvl, KM_SLEEP); + kmem_free(payload, payloadlen); + if (err != 0) + goto out; + } + + if (featureflags & DMU_BACKUP_FEATURE_RESUMING) { + err = resume_check(ra, begin_nvl); + if (err != 0) + goto out; + } (void) bqueue_init(&rwa->q, zfs_recv_queue_length, offsetof(struct receive_record_arg, node)); @@ -2425,6 +2908,7 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp, mutex_init(&rwa->mutex, NULL, MUTEX_DEFAULT, NULL); rwa->os = ra->os; rwa->byteswap = drc->drc_byteswap; + rwa->resumable = drc->drc_resumable; (void) thread_create(NULL, 0, receive_writer_thread, rwa, 0, curproc, TS_RUN, minclsyspri); @@ -2438,7 +2922,7 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp, * We can leave this loop in 3 ways: First, if rwa->err is * non-zero. In that case, the writer thread will free the rrd we just * pushed. Second, if we're interrupted; in that case, either it's the - * first loop and ra->rrd was never allocated, or it's later, and ra.rrd + * first loop and ra->rrd was never allocated, or it's later and ra->rrd * has been handed off to the writer thread who will free it. Finally, * if receive_read_record fails or we're at the end of the stream, then * we free ra->rrd and exit. @@ -2483,24 +2967,21 @@ dmu_recv_stream(dmu_recv_cookie_t *drc, vnode_t *vp, offset_t *voffp, err = rwa->err; out: + nvlist_free(begin_nvl); if ((featureflags & DMU_BACKUP_FEATURE_DEDUP) && (cleanup_fd != -1)) zfs_onexit_fd_rele(cleanup_fd); if (err != 0) { /* - * destroy what we created, so we don't leave it in the - * inconsistent restoring state. + * Clean up references. If receive is not resumable, + * destroy what we created, so we don't leave it in + * the inconsistent state. */ dmu_recv_cleanup_ds(drc); } *voffp = ra->voff; - - for (n = list_remove_head(&ra->ignore_obj_list); n != NULL; - n = list_remove_head(&ra->ignore_obj_list)) { - kmem_free(n, sizeof (*n)); - } - list_destroy(&ra->ignore_obj_list); + objlist_destroy(&ra->ignore_objlist); kmem_free(ra, sizeof (*ra)); kmem_free(rwa, sizeof (*rwa)); return (err); @@ -2651,6 +3132,20 @@ dmu_recv_end_sync(void *arg, dmu_tx_t *tx) dmu_buf_will_dirty(ds->ds_dbuf, tx); dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT; + if (dsl_dataset_has_resume_receive_state(ds)) { + (void) zap_remove(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_FROMGUID, tx); + (void) zap_remove(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_OBJECT, tx); + (void) zap_remove(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_OFFSET, tx); + (void) zap_remove(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_BYTES, tx); + (void) zap_remove(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_TOGUID, tx); + (void) zap_remove(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_TONAME, tx); + } } drc->drc_newsnapobj = dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj; zvol_create_minors(dp->dp_spa, drc->drc_tofs, B_TRUE); @@ -2699,16 +3194,13 @@ dmu_recv_existing_end(dmu_recv_cookie_t *drc) int error; #ifdef _KERNEL - char *name; - /* * We will be destroying the ds; make sure its origin is unmounted if * necessary. */ - name = kmem_alloc(MAXNAMELEN, KM_SLEEP); + char name[ZFS_MAX_DATASET_NAME_LEN]; dsl_dataset_name(drc->drc_ds, name); zfs_destroy_unmount_origin(name); - kmem_free(name, MAXNAMELEN); #endif error = dsl_sync_task(drc->drc_tofs, diff --git a/module/zfs/dmu_traverse.c b/module/zfs/dmu_traverse.c index bba9efe14c7a..0df12fac8c36 100644 --- a/module/zfs/dmu_traverse.c +++ b/module/zfs/dmu_traverse.c @@ -47,6 +47,7 @@ typedef struct prefetch_data { int pd_flags; boolean_t pd_cancel; boolean_t pd_exited; + zbookmark_phys_t pd_resume; } prefetch_data_t; typedef struct traverse_data { @@ -323,30 +324,29 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp, uint32_t flags = ARC_FLAG_WAIT; int32_t i; int32_t epb = BP_GET_LSIZE(bp) >> DNODE_SHIFT; - dnode_phys_t *cdnp; + dnode_phys_t *child_dnp; err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf, ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb); if (err != 0) goto post; - cdnp = buf->b_data; + child_dnp = buf->b_data; - for (i = 0; i < epb; i++) { - prefetch_dnode_metadata(td, &cdnp[i], zb->zb_objset, - zb->zb_blkid * epb + i); + for (i = 0; i < epb; i += child_dnp[i].dn_extra_slots + 1) { + prefetch_dnode_metadata(td, &child_dnp[i], + zb->zb_objset, zb->zb_blkid * epb + i); } /* recursively visitbp() blocks below this */ - for (i = 0; i < epb; i++) { - err = traverse_dnode(td, &cdnp[i], zb->zb_objset, - zb->zb_blkid * epb + i); + for (i = 0; i < epb; i += child_dnp[i].dn_extra_slots + 1) { + err = traverse_dnode(td, &child_dnp[i], + zb->zb_objset, zb->zb_blkid * epb + i); if (err != 0) break; } } else if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) { arc_flags_t flags = ARC_FLAG_WAIT; objset_phys_t *osp; - dnode_phys_t *mdnp, *gdnp, *udnp; err = arc_read(NULL, td->td_spa, bp, arc_getbuf_func, &buf, ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL, &flags, zb); @@ -354,11 +354,7 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp, goto post; osp = buf->b_data; - mdnp = &osp->os_meta_dnode; - gdnp = &osp->os_groupused_dnode; - udnp = &osp->os_userused_dnode; - - prefetch_dnode_metadata(td, mdnp, zb->zb_objset, + prefetch_dnode_metadata(td, &osp->os_meta_dnode, zb->zb_objset, DMU_META_DNODE_OBJECT); /* * See the block comment above for the goal of this variable. @@ -370,21 +366,21 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp, td->td_realloc_possible = B_FALSE; if (arc_buf_size(buf) >= sizeof (objset_phys_t)) { - prefetch_dnode_metadata(td, gdnp, zb->zb_objset, - DMU_GROUPUSED_OBJECT); - prefetch_dnode_metadata(td, udnp, zb->zb_objset, - DMU_USERUSED_OBJECT); + prefetch_dnode_metadata(td, &osp->os_groupused_dnode, + zb->zb_objset, DMU_GROUPUSED_OBJECT); + prefetch_dnode_metadata(td, &osp->os_userused_dnode, + zb->zb_objset, DMU_USERUSED_OBJECT); } - err = traverse_dnode(td, mdnp, zb->zb_objset, + err = traverse_dnode(td, &osp->os_meta_dnode, zb->zb_objset, DMU_META_DNODE_OBJECT); if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) { - err = traverse_dnode(td, gdnp, zb->zb_objset, - DMU_GROUPUSED_OBJECT); + err = traverse_dnode(td, &osp->os_groupused_dnode, + zb->zb_objset, DMU_GROUPUSED_OBJECT); } if (err == 0 && arc_buf_size(buf) >= sizeof (objset_phys_t)) { - err = traverse_dnode(td, udnp, zb->zb_objset, - DMU_USERUSED_OBJECT); + err = traverse_dnode(td, &osp->os_userused_dnode, + zb->zb_objset, DMU_USERUSED_OBJECT); } } @@ -416,9 +412,15 @@ traverse_visitbp(traverse_data_t *td, const dnode_phys_t *dnp, * Set the bookmark to the first level-0 block that we need * to visit. This way, the resuming code does not need to * deal with resuming from indirect blocks. + * + * Note, if zb_level <= 0, dnp may be NULL, so we don't want + * to dereference it. */ - td->td_resume->zb_blkid = zb->zb_blkid << - (zb->zb_level * (dnp->dn_indblkshift - SPA_BLKPTRSHIFT)); + td->td_resume->zb_blkid = zb->zb_blkid; + if (zb->zb_level > 0) { + td->td_resume->zb_blkid <<= zb->zb_level * + (dnp->dn_indblkshift - SPA_BLKPTRSHIFT); + } td->td_paused = B_TRUE; } @@ -439,7 +441,7 @@ prefetch_dnode_metadata(traverse_data_t *td, const dnode_phys_t *dnp, if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { SET_BOOKMARK(&czb, objset, object, 0, DMU_SPILL_BLKID); - traverse_prefetch_metadata(td, &dnp->dn_spill, &czb); + traverse_prefetch_metadata(td, DN_SPILL_BLKPTR(dnp), &czb); } } @@ -450,6 +452,10 @@ traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp, int j, err = 0; zbookmark_phys_t czb; + if (object != DMU_META_DNODE_OBJECT && td->td_resume != NULL && + object < td->td_resume->zb_object) + return (0); + if (td->td_flags & TRAVERSE_PRE) { SET_BOOKMARK(&czb, objset, object, ZB_DNODE_LEVEL, ZB_DNODE_BLKID); @@ -470,7 +476,7 @@ traverse_dnode(traverse_data_t *td, const dnode_phys_t *dnp, if (err == 0 && (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) { SET_BOOKMARK(&czb, objset, object, 0, DMU_SPILL_BLKID); - err = traverse_visitbp(td, dnp, &dnp->dn_spill, &czb); + err = traverse_visitbp(td, dnp, DN_SPILL_BLKPTR(dnp), &czb); } if (err == 0 && (td->td_flags & TRAVERSE_POST)) { @@ -527,6 +533,7 @@ traverse_prefetch_thread(void *arg) td.td_func = traverse_prefetcher; td.td_arg = td_main->td_pfd; td.td_pfd = NULL; + td.td_resume = &td_main->td_pfd->pd_resume; SET_BOOKMARK(&czb, td.td_objset, ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID); @@ -556,12 +563,6 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp, ASSERT(ds == NULL || objset == ds->ds_object); ASSERT(!(flags & TRAVERSE_PRE) || !(flags & TRAVERSE_POST)); - /* - * The data prefetching mechanism (the prefetch thread) is incompatible - * with resuming from a bookmark. - */ - ASSERT(resume == NULL || !(flags & TRAVERSE_PREFETCH_DATA)); - td = kmem_alloc(sizeof (traverse_data_t), KM_SLEEP); pd = kmem_zalloc(sizeof (prefetch_data_t), KM_SLEEP); czb = kmem_alloc(sizeof (zbookmark_phys_t), KM_SLEEP); @@ -586,6 +587,8 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp, } pd->pd_flags = flags; + if (resume != NULL) + pd->pd_resume = *resume; mutex_init(&pd->pd_mtx, NULL, MUTEX_DEFAULT, NULL); cv_init(&pd->pd_cv, NULL, CV_DEFAULT, NULL); @@ -638,11 +641,19 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp, * in syncing context). */ int -traverse_dataset(dsl_dataset_t *ds, uint64_t txg_start, int flags, - blkptr_cb_t func, void *arg) +traverse_dataset_resume(dsl_dataset_t *ds, uint64_t txg_start, + zbookmark_phys_t *resume, + int flags, blkptr_cb_t func, void *arg) { return (traverse_impl(ds->ds_dir->dd_pool->dp_spa, ds, ds->ds_object, - &dsl_dataset_phys(ds)->ds_bp, txg_start, NULL, flags, func, arg)); + &dsl_dataset_phys(ds)->ds_bp, txg_start, resume, flags, func, arg)); +} + +int +traverse_dataset(dsl_dataset_t *ds, uint64_t txg_start, + int flags, blkptr_cb_t func, void *arg) +{ + return (traverse_dataset_resume(ds, txg_start, NULL, flags, func, arg)); } int @@ -675,7 +686,7 @@ traverse_pool(spa_t *spa, uint64_t txg_start, int flags, /* visit each dataset */ for (obj = 1; err == 0; - err = dmu_object_next(mos, &obj, FALSE, txg_start)) { + err = dmu_object_next(mos, &obj, B_FALSE, txg_start)) { dmu_object_info_t doi; err = dmu_object_info(mos, obj, &doi); diff --git a/module/zfs/dmu_tx.c b/module/zfs/dmu_tx.c index 74e323dbdf65..ed29bfbc692b 100644 --- a/module/zfs/dmu_tx.c +++ b/module/zfs/dmu_tx.c @@ -1586,7 +1586,7 @@ dmu_tx_hold_spill(dmu_tx_t *tx, uint64_t object) } else { blkptr_t *bp; - bp = &dn->dn_phys->dn_spill; + bp = DN_SPILL_BLKPTR(dn->dn_phys); if (dsl_dataset_block_freeable(dn->dn_objset->os_dsl_dataset, bp, bp->blk_birth)) txh->txh_space_tooverwrite += SPA_OLD_MAXBLOCKSIZE; @@ -1618,7 +1618,7 @@ dmu_tx_hold_sa_create(dmu_tx_t *tx, int attrsize) dmu_tx_sa_registration_hold(sa, tx); - if (attrsize <= DN_MAX_BONUSLEN && !sa->sa_force_spill) + if (attrsize <= DN_OLD_MAX_BONUSLEN && !sa->sa_force_spill) return; (void) dmu_tx_hold_object_impl(tx, tx->tx_objset, DMU_NEW_OBJECT, diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c index 38bcecd467e3..8015f54edc56 100644 --- a/module/zfs/dnode.c +++ b/module/zfs/dnode.c @@ -248,6 +248,7 @@ dnode_verify(dnode_t *dn) } if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) { int i; + int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots); ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT); if (dn->dn_datablkshift) { ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT); @@ -258,12 +259,12 @@ dnode_verify(dnode_t *dn) ASSERT(DMU_OT_IS_VALID(dn->dn_type)); ASSERT3U(dn->dn_nblkptr, >=, 1); ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); - ASSERT3U(dn->dn_bonuslen, <=, DN_MAX_BONUSLEN); + ASSERT3U(dn->dn_bonuslen, <=, max_bonuslen); ASSERT3U(dn->dn_datablksz, ==, dn->dn_datablkszsec << SPA_MINBLOCKSHIFT); ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0); ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) + - dn->dn_bonuslen, <=, DN_MAX_BONUSLEN); + dn->dn_bonuslen, <=, max_bonuslen); for (i = 0; i < TXG_SIZE; i++) { ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels); } @@ -294,6 +295,7 @@ dnode_byteswap(dnode_phys_t *dnp) dnp->dn_datablkszsec = BSWAP_16(dnp->dn_datablkszsec); dnp->dn_bonuslen = BSWAP_16(dnp->dn_bonuslen); + dnp->dn_extra_slots = BSWAP_8(dnp->dn_extra_slots); dnp->dn_maxblkid = BSWAP_64(dnp->dn_maxblkid); dnp->dn_used = BSWAP_64(dnp->dn_used); @@ -320,7 +322,8 @@ dnode_byteswap(dnode_phys_t *dnp) * dnode buffer). */ int off = (dnp->dn_nblkptr-1) * sizeof (blkptr_t); - size_t len = DN_MAX_BONUSLEN - off; + int slots = dnp->dn_extra_slots + 1; + size_t len = DN_SLOTS_TO_BONUSLEN(slots) - off; dmu_object_byteswap_t byteswap; ASSERT(DMU_OT_IS_VALID(dnp->dn_bonustype)); byteswap = DMU_OT_BYTESWAP(dnp->dn_bonustype); @@ -329,23 +332,24 @@ dnode_byteswap(dnode_phys_t *dnp) /* Swap SPILL block if we have one */ if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) - byteswap_uint64_array(&dnp->dn_spill, sizeof (blkptr_t)); - + byteswap_uint64_array(DN_SPILL_BLKPTR(dnp), sizeof (blkptr_t)); } void dnode_buf_byteswap(void *vbuf, size_t size) { - dnode_phys_t *buf = vbuf; - int i; + int i = 0; ASSERT3U(sizeof (dnode_phys_t), ==, (1<>= DNODE_SHIFT; - for (i = 0; i < size; i++) { - dnode_byteswap(buf); - buf++; + while (i < size) { + dnode_phys_t *dnp = vbuf + i; + dnode_byteswap(dnp); + + i += DNODE_MIN_SIZE; + if (dnp->dn_type != DMU_OT_NONE) + i += dnp->dn_extra_slots * DNODE_MIN_SIZE; } } @@ -356,7 +360,7 @@ dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx) dnode_setdirty(dn, tx); rw_enter(&dn->dn_struct_rwlock, RW_WRITER); - ASSERT3U(newsize, <=, DN_MAX_BONUSLEN - + ASSERT3U(newsize, <=, DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - (dn->dn_nblkptr-1) * sizeof (blkptr_t)); dn->dn_bonuslen = newsize; if (newsize == 0) @@ -434,6 +438,7 @@ dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db, dn->dn_compress = dnp->dn_compress; dn->dn_bonustype = dnp->dn_bonustype; dn->dn_bonuslen = dnp->dn_bonuslen; + dn->dn_num_slots = dnp->dn_extra_slots + 1; dn->dn_maxblkid = dnp->dn_maxblkid; dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0); dn->dn_id_flags = 0; @@ -470,7 +475,7 @@ dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db, dnh->dnh_dnode = dn; mutex_exit(&os->os_lock); - arc_space_consume(sizeof (dnode_t), ARC_SPACE_OTHER); + arc_space_consume(sizeof (dnode_t), ARC_SPACE_DNODE); return (dn); } @@ -526,7 +531,7 @@ dnode_destroy(dnode_t *dn) dmu_zfetch_fini(&dn->dn_zfetch); kmem_cache_free(dnode_cache, dn); - arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER); + arc_space_return(sizeof (dnode_t), ARC_SPACE_DNODE); if (complete_os_eviction) dmu_objset_evict_done(os); @@ -534,10 +539,13 @@ dnode_destroy(dnode_t *dn) void dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, - dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) + dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx) { int i; + ASSERT3U(dn_slots, >, 0); + ASSERT3U(dn_slots << DNODE_SHIFT, <=, + spa_maxdnodesize(dmu_objset_spa(dn->dn_objset))); ASSERT3U(blocksize, <=, spa_maxblocksize(dmu_objset_spa(dn->dn_objset))); if (blocksize == 0) @@ -550,8 +558,8 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, ibs = MIN(MAX(ibs, DN_MIN_INDBLKSHIFT), DN_MAX_INDBLKSHIFT); - dprintf("os=%p obj=%llu txg=%llu blocksize=%d ibs=%d\n", dn->dn_objset, - dn->dn_object, tx->tx_txg, blocksize, ibs); + dprintf("os=%p obj=%llu txg=%llu blocksize=%d ibs=%d dn_slots=%d\n", + dn->dn_objset, dn->dn_object, tx->tx_txg, blocksize, ibs, dn_slots); ASSERT(dn->dn_type == DMU_OT_NONE); ASSERT(bcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t)) == 0); @@ -562,7 +570,7 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, (bonustype == DMU_OT_SA && bonuslen == 0) || (bonustype != DMU_OT_NONE && bonuslen != 0)); ASSERT(DMU_OT_IS_VALID(bonustype)); - ASSERT3U(bonuslen, <=, DN_MAX_BONUSLEN); + ASSERT3U(bonuslen, <=, DN_SLOTS_TO_BONUSLEN(dn_slots)); ASSERT(dn->dn_type == DMU_OT_NONE); ASSERT0(dn->dn_maxblkid); ASSERT0(dn->dn_allocated_txg); @@ -588,11 +596,15 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, dnode_setdblksz(dn, blocksize); dn->dn_indblkshift = ibs; dn->dn_nlevels = 1; + dn->dn_num_slots = dn_slots; if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */ dn->dn_nblkptr = 1; - else - dn->dn_nblkptr = 1 + - ((DN_MAX_BONUSLEN - bonuslen) >> SPA_BLKPTRSHIFT); + else { + dn->dn_nblkptr = MIN(DN_MAX_NBLKPTR, + 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >> + SPA_BLKPTRSHIFT)); + } + dn->dn_bonustype = bonustype; dn->dn_bonuslen = bonuslen; dn->dn_checksum = ZIO_CHECKSUM_INHERIT; @@ -617,7 +629,7 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, void dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, - dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) + dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx) { int nblkptr; @@ -631,7 +643,10 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, (bonustype != DMU_OT_NONE && bonuslen != 0) || (bonustype == DMU_OT_SA && bonuslen == 0)); ASSERT(DMU_OT_IS_VALID(bonustype)); - ASSERT3U(bonuslen, <=, DN_MAX_BONUSLEN); + ASSERT3U(bonuslen, <=, + DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(dn->dn_objset)))); + + dn_slots = dn_slots > 0 ? dn_slots : DNODE_MIN_SLOTS; /* clean up any unreferenced dbufs */ dnode_evict_dbufs(dn); @@ -654,7 +669,9 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */ nblkptr = 1; else - nblkptr = 1 + ((DN_MAX_BONUSLEN - bonuslen) >> SPA_BLKPTRSHIFT); + nblkptr = MIN(DN_MAX_NBLKPTR, + 1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >> + SPA_BLKPTRSHIFT)); if (dn->dn_bonustype != bonustype) dn->dn_next_bonustype[tx->tx_txg&TXG_MASK] = bonustype; if (dn->dn_nblkptr != nblkptr) @@ -672,6 +689,7 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, mutex_enter(&dn->dn_mtx); dn->dn_bonustype = bonustype; dn->dn_bonuslen = bonuslen; + dn->dn_num_slots = dn_slots; dn->dn_nblkptr = nblkptr; dn->dn_checksum = ZIO_CHECKSUM_INHERIT; dn->dn_compress = ZIO_COMPRESS_INHERIT; @@ -680,7 +698,8 @@ dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, /* fix up the bonus db_size */ if (dn->dn_bonus) { dn->dn_bonus->db.db_size = - DN_MAX_BONUSLEN - (dn->dn_nblkptr-1) * sizeof (blkptr_t); + DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) - + (dn->dn_nblkptr-1) * sizeof (blkptr_t); ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size); } @@ -1052,25 +1071,151 @@ dnode_buf_pageout(void *dbu) children_dnodes->dnc_count * sizeof (dnode_handle_t)); } +/* + * Return true if the given index is interior to a dnode already + * allocated in the block. That is, the index is neither free nor + * allocated, but is consumed by a large dnode. + * + * The dnode_phys_t buffer may not be in sync with the in-core dnode + * structure, so we try to check the dnode structure first and fall back + * to the dnode_phys_t buffer it doesn't exist. + */ +static boolean_t +dnode_is_consumed(dmu_buf_impl_t *db, int idx) +{ + dnode_handle_t *dnh; + dmu_object_type_t ot; + dnode_children_t *children_dnodes; + dnode_phys_t *dn_block; + int skip; + int i; + + children_dnodes = dmu_buf_get_user(&db->db); + dn_block = (dnode_phys_t *)db->db.db_data; + + for (i = 0; i < idx; i += skip) { + dnh = &children_dnodes->dnc_children[i]; + + zrl_add(&dnh->dnh_zrlock); + if (dnh->dnh_dnode != NULL) { + ot = dnh->dnh_dnode->dn_type; + skip = dnh->dnh_dnode->dn_num_slots; + } else { + ot = dn_block[i].dn_type; + skip = dn_block[i].dn_extra_slots + 1; + } + zrl_remove(&dnh->dnh_zrlock); + + if (ot == DMU_OT_NONE) + skip = 1; + } + + return (i > idx); +} + +/* + * Return true if the given index in the dnode block is a valid + * allocated dnode. That is, the index is not consumed by a large + * dnode and is not free. + * + * The dnode_phys_t buffer may not be in sync with the in-core dnode + * structure, so we try to check the dnode structure first and fall back + * to the dnode_phys_t buffer it doesn't exist. + */ +static boolean_t +dnode_is_allocated(dmu_buf_impl_t *db, int idx) +{ + dnode_handle_t *dnh; + dmu_object_type_t ot; + dnode_children_t *children_dnodes; + dnode_phys_t *dn_block; + + if (dnode_is_consumed(db, idx)) + return (B_FALSE); + + children_dnodes = dmu_buf_get_user(&db->db); + dn_block = (dnode_phys_t *)db->db.db_data; + + dnh = &children_dnodes->dnc_children[idx]; + + zrl_add(&dnh->dnh_zrlock); + if (dnh->dnh_dnode != NULL) + ot = dnh->dnh_dnode->dn_type; + else + ot = dn_block[idx].dn_type; + zrl_remove(&dnh->dnh_zrlock); + + return (ot != DMU_OT_NONE); +} + +/* + * Return true if the given range of indices in the dnode block are + * free. That is, the starting index is not consumed by a large dnode + * and none of the indices are allocated. + * + * The dnode_phys_t buffer may not be in sync with the in-core dnode + * structure, so we try to check the dnode structure first and fall back + * to the dnode_phys_t buffer it doesn't exist. + */ +static boolean_t +dnode_is_free(dmu_buf_impl_t *db, int idx, int slots) +{ + dnode_handle_t *dnh; + dmu_object_type_t ot; + dnode_children_t *children_dnodes; + dnode_phys_t *dn_block; + int i; + + if (idx + slots > DNODES_PER_BLOCK) + return (B_FALSE); + + children_dnodes = dmu_buf_get_user(&db->db); + dn_block = (dnode_phys_t *)db->db.db_data; + + if (dnode_is_consumed(db, idx)) + return (B_FALSE); + + for (i = idx; i < idx + slots; i++) { + dnh = &children_dnodes->dnc_children[i]; + + zrl_add(&dnh->dnh_zrlock); + if (dnh->dnh_dnode != NULL) + ot = dnh->dnh_dnode->dn_type; + else + ot = dn_block[i].dn_type; + zrl_remove(&dnh->dnh_zrlock); + + if (ot != DMU_OT_NONE) + return (B_FALSE); + } + + return (B_TRUE); +} + /* * errors: * EINVAL - invalid object number. + * ENOSPC - hole too small to fulfill "slots" request * EIO - i/o error. * succeeds even for free dnodes. */ int -dnode_hold_impl(objset_t *os, uint64_t object, int flag, +dnode_hold_impl(objset_t *os, uint64_t object, int flag, int slots, void *tag, dnode_t **dnp) { - int epb, idx, err; + int epb, idx, err, i; int drop_struct_lock = FALSE; int type; uint64_t blk; dnode_t *mdn, *dn; dmu_buf_impl_t *db; dnode_children_t *children_dnodes; + dnode_phys_t *dn_block_begin; dnode_handle_t *dnh; + ASSERT(!(flag & DNODE_MUST_BE_ALLOCATED) || (slots == 0)); + ASSERT(!(flag & DNODE_MUST_BE_FREE) || (slots > 0)); + /* * If you are holding the spa config lock as writer, you shouldn't * be asking the DMU to do *anything* unless it's the root pool @@ -1126,12 +1271,9 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag, ASSERT3U(db->db.db_size, >=, 1<db.db_size >> DNODE_SHIFT; - idx = object & (epb-1); - ASSERT(DB_DNODE(db)->dn_type == DMU_OT_DNODE); children_dnodes = dmu_buf_get_user(&db->db); if (children_dnodes == NULL) { - int i; dnode_children_t *winner; children_dnodes = kmem_zalloc(sizeof (dnode_children_t) + epb * sizeof (dnode_handle_t), KM_SLEEP); @@ -1156,21 +1298,28 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag, } ASSERT(children_dnodes->dnc_count == epb); + idx = object & (epb - 1); + dn_block_begin = (dnode_phys_t *)db->db.db_data; + + if ((flag & DNODE_MUST_BE_FREE) && !dnode_is_free(db, idx, slots)) { + dbuf_rele(db, FTAG); + return (ENOSPC); + } else if ((flag & DNODE_MUST_BE_ALLOCATED) && + !dnode_is_allocated(db, idx)) { + dbuf_rele(db, FTAG); + return (ENOENT); + } + dnh = &children_dnodes->dnc_children[idx]; zrl_add(&dnh->dnh_zrlock); dn = dnh->dnh_dnode; - if (dn == NULL) { - dnode_phys_t *phys = (dnode_phys_t *)db->db.db_data+idx; - - dn = dnode_create(os, phys, db, object, dnh); - } + if (dn == NULL) + dn = dnode_create(os, dn_block_begin + idx, db, object, dnh); mutex_enter(&dn->dn_mtx); type = dn->dn_type; if (dn->dn_free_txg || - ((flag & DNODE_MUST_BE_ALLOCATED) && type == DMU_OT_NONE) || - ((flag & DNODE_MUST_BE_FREE) && - (type != DMU_OT_NONE || !refcount_is_zero(&dn->dn_holds)))) { + ((flag & DNODE_MUST_BE_FREE) && !refcount_is_zero(&dn->dn_holds))) { mutex_exit(&dn->dn_mtx); zrl_remove(&dnh->dnh_zrlock); dbuf_rele(db, FTAG); @@ -1198,7 +1347,8 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag, int dnode_hold(objset_t *os, uint64_t object, void *tag, dnode_t **dnp) { - return (dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, tag, dnp)); + return (dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, 0, tag, + dnp)); } /* @@ -1908,17 +2058,21 @@ dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset, error = SET_ERROR(ESRCH); } else if (lvl == 0) { dnode_phys_t *dnp = data; - span = DNODE_SHIFT; + ASSERT(dn->dn_type == DMU_OT_DNODE); + ASSERT(!(flags & DNODE_FIND_BACKWARDS)); - for (i = (*offset >> span) & (blkfill - 1); - i >= 0 && i < blkfill; i += inc) { + for (i = (*offset >> DNODE_SHIFT) & (blkfill - 1); + i < blkfill; i += dnp[i].dn_extra_slots + 1) { if ((dnp[i].dn_type == DMU_OT_NONE) == hole) break; - *offset += (1ULL << span) * inc; } - if (i < 0 || i == blkfill) + + if (i == blkfill) error = SET_ERROR(ESRCH); + + *offset = (*offset & ~(DNODE_BLOCK_SIZE - 1)) + + (i << DNODE_SHIFT); } else { blkptr_t *bp = data; uint64_t start = *offset; diff --git a/module/zfs/dnode_sync.c b/module/zfs/dnode_sync.c index b47395a1e26a..54066e2e3ffa 100644 --- a/module/zfs/dnode_sync.c +++ b/module/zfs/dnode_sync.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2015 by Delphix. All rights reserved. + * Copyright (c) 2012, 2016 by Delphix. All rights reserved. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. */ @@ -60,20 +60,14 @@ dnode_increase_indirection(dnode_t *dn, dmu_tx_t *tx) dprintf("os=%p obj=%llu, increase to %d\n", dn->dn_objset, dn->dn_object, dn->dn_phys->dn_nlevels); - /* check for existing blkptrs in the dnode */ - for (i = 0; i < nblkptr; i++) - if (!BP_IS_HOLE(&dn->dn_phys->dn_blkptr[i])) - break; - if (i != nblkptr) { - /* transfer dnode's block pointers to new indirect block */ - (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED|DB_RF_HAVESTRUCT); - ASSERT(db->db.db_data); - ASSERT(arc_released(db->db_buf)); - ASSERT3U(sizeof (blkptr_t) * nblkptr, <=, db->db.db_size); - bcopy(dn->dn_phys->dn_blkptr, db->db.db_data, - sizeof (blkptr_t) * nblkptr); - arc_buf_freeze(db->db_buf); - } + /* transfer dnode's block pointers to new indirect block */ + (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED|DB_RF_HAVESTRUCT); + ASSERT(db->db.db_data); + ASSERT(arc_released(db->db_buf)); + ASSERT3U(sizeof (blkptr_t) * nblkptr, <=, db->db.db_size); + bcopy(dn->dn_phys->dn_blkptr, db->db.db_data, + sizeof (blkptr_t) * nblkptr); + arc_buf_freeze(db->db_buf); /* set dbuf's parent pointers to new indirect buf */ for (i = 0; i < nblkptr; i++) { @@ -530,7 +524,7 @@ dnode_sync_free(dnode_t *dn, dmu_tx_t *tx) ASSERT(dn->dn_free_txg > 0); if (dn->dn_allocated_txg != dn->dn_free_txg) dmu_buf_will_dirty(&dn->dn_dbuf->db, tx); - bzero(dn->dn_phys, sizeof (dnode_phys_t)); + bzero(dn->dn_phys, sizeof (dnode_phys_t) * dn->dn_num_slots); mutex_enter(&dn->dn_mtx); dn->dn_type = DMU_OT_NONE; @@ -565,7 +559,7 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx) ASSERT(dmu_tx_is_syncing(tx)); ASSERT(dnp->dn_type != DMU_OT_NONE || dn->dn_allocated_txg); ASSERT(dnp->dn_type != DMU_OT_NONE || - bcmp(dnp, &zerodn, DNODE_SIZE) == 0); + bcmp(dnp, &zerodn, DNODE_MIN_SIZE) == 0); DNODE_VERIFY(dn); ASSERT(dn->dn_dbuf == NULL || arc_released(dn->dn_dbuf->db_buf)); @@ -597,6 +591,9 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx) dnp->dn_bonustype = dn->dn_bonustype; dnp->dn_bonuslen = dn->dn_bonuslen; } + + dnp->dn_extra_slots = dn->dn_num_slots - 1; + ASSERT(dnp->dn_nlevels > 1 || BP_IS_HOLE(&dnp->dn_blkptr[0]) || BP_IS_EMBEDDED(&dnp->dn_blkptr[0]) || @@ -629,7 +626,8 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx) dnp->dn_bonuslen = 0; else dnp->dn_bonuslen = dn->dn_next_bonuslen[txgoff]; - ASSERT(dnp->dn_bonuslen <= DN_MAX_BONUSLEN); + ASSERT(dnp->dn_bonuslen <= + DN_SLOTS_TO_BONUSLEN(dnp->dn_extra_slots + 1)); dn->dn_next_bonuslen[txgoff] = 0; } @@ -668,7 +666,7 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx) mutex_exit(&dn->dn_mtx); if (kill_spill) { - free_blocks(dn, &dn->dn_phys->dn_spill, 1, tx); + free_blocks(dn, DN_SPILL_BLKPTR(dn->dn_phys), 1, tx); mutex_enter(&dn->dn_mtx); dnp->dn_flags &= ~DNODE_FLAG_SPILL_BLKPTR; mutex_exit(&dn->dn_mtx); @@ -688,10 +686,19 @@ dnode_sync(dnode_t *dn, dmu_tx_t *tx) } if (freeing_dnode) { + dn->dn_objset->os_freed_dnodes++; dnode_sync_free(dn, tx); return; } + if (dn->dn_num_slots > DNODE_MIN_SLOTS) { + dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; + mutex_enter(&ds->ds_lock); + ds->ds_feature_activation_needed[SPA_FEATURE_LARGE_DNODE] = + B_TRUE; + mutex_exit(&ds->ds_lock); + } + if (dn->dn_next_nlevels[txgoff]) { dnode_increase_indirection(dn, tx); dn->dn_next_nlevels[txgoff] = 0; diff --git a/module/zfs/dsl_bookmark.c b/module/zfs/dsl_bookmark.c index 447a3a2dc3a2..5a7f034ce649 100644 --- a/module/zfs/dsl_bookmark.c +++ b/module/zfs/dsl_bookmark.c @@ -34,10 +34,10 @@ static int dsl_bookmark_hold_ds(dsl_pool_t *dp, const char *fullname, dsl_dataset_t **dsp, void *tag, char **shortnamep) { - char buf[MAXNAMELEN]; + char buf[ZFS_MAX_DATASET_NAME_LEN]; char *hashp; - if (strlen(fullname) >= MAXNAMELEN) + if (strlen(fullname) >= ZFS_MAX_DATASET_NAME_LEN) return (SET_ERROR(ENAMETOOLONG)); hashp = strchr(fullname, '#'); if (hashp == NULL) diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c index 230027daf995..1b2ac72b04f3 100644 --- a/module/zfs/dsl_dataset.c +++ b/module/zfs/dsl_dataset.c @@ -25,6 +25,7 @@ * Copyright (c) 2014 RackTop Systems. * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved. * Copyright (c) 2016 Actifio, Inc. All rights reserved. + * Copyright 2016, OmniTI Computer Consulting, Inc. All rights reserved. */ #include @@ -51,6 +52,10 @@ #include #include #include +#include +#include +#include +#include /* * The SPA supports block sizes up to 16MB. However, very large blocks @@ -74,6 +79,8 @@ int zfs_max_recordsize = 1 * 1024 * 1024; extern inline dsl_dataset_phys_t *dsl_dataset_phys(dsl_dataset_t *ds); +extern int spa_asize_inflation; + /* * Figure out how much of this delta should be propogated to the dsl_dir * layer. If there's a refreservation, that space has already been @@ -663,22 +670,38 @@ dsl_dataset_name(dsl_dataset_t *ds, char *name) dsl_dir_name(ds->ds_dir, name); VERIFY0(dsl_dataset_get_snapname(ds)); if (ds->ds_snapname[0]) { - (void) strcat(name, "@"); + VERIFY3U(strlcat(name, "@", ZFS_MAX_DATASET_NAME_LEN), + <, ZFS_MAX_DATASET_NAME_LEN); /* * We use a "recursive" mutex so that we * can call dprintf_ds() with ds_lock held. */ if (!MUTEX_HELD(&ds->ds_lock)) { mutex_enter(&ds->ds_lock); - (void) strcat(name, ds->ds_snapname); + VERIFY3U(strlcat(name, ds->ds_snapname, + ZFS_MAX_DATASET_NAME_LEN), <, + ZFS_MAX_DATASET_NAME_LEN); mutex_exit(&ds->ds_lock); } else { - (void) strcat(name, ds->ds_snapname); + VERIFY3U(strlcat(name, ds->ds_snapname, + ZFS_MAX_DATASET_NAME_LEN), <, + ZFS_MAX_DATASET_NAME_LEN); } } } } +int +dsl_dataset_namelen(dsl_dataset_t *ds) +{ + int len; + VERIFY0(dsl_dataset_get_snapname(ds)); + mutex_enter(&ds->ds_lock); + len = dsl_dir_namelen(ds->ds_dir) + 1 + strlen(ds->ds_snapname); + mutex_exit(&ds->ds_lock); + return (len); +} + void dsl_dataset_rele(dsl_dataset_t *ds, void *tag) { @@ -703,6 +726,7 @@ dsl_dataset_tryown(dsl_dataset_t *ds, void *tag) { boolean_t gotit = FALSE; + ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool)); mutex_enter(&ds->ds_lock); if (ds->ds_owner == NULL && !DS_IS_INCONSISTENT(ds)) { ds->ds_owner = tag; @@ -713,6 +737,16 @@ dsl_dataset_tryown(dsl_dataset_t *ds, void *tag) return (gotit); } +boolean_t +dsl_dataset_has_owner(dsl_dataset_t *ds) +{ + boolean_t rv; + mutex_enter(&ds->ds_lock); + rv = (ds->ds_owner != NULL); + mutex_exit(&ds->ds_lock); + return (rv); +} + static void dsl_dataset_activate_feature(uint64_t dsobj, spa_feature_t f, dmu_tx_t *tx) { @@ -1237,10 +1271,10 @@ dsl_dataset_snapshot_check(void *arg, dmu_tx_t *tx) int error = 0; dsl_dataset_t *ds; char *name, *atp; - char dsname[MAXNAMELEN]; + char dsname[ZFS_MAX_DATASET_NAME_LEN]; name = nvpair_name(pair); - if (strlen(name) >= MAXNAMELEN) + if (strlen(name) >= ZFS_MAX_DATASET_NAME_LEN) error = SET_ERROR(ENAMETOOLONG); if (error == 0) { atp = strchr(name, '@'); @@ -1413,7 +1447,7 @@ dsl_dataset_snapshot_sync(void *arg, dmu_tx_t *tx) pair != NULL; pair = nvlist_next_nvpair(ddsa->ddsa_snaps, pair)) { dsl_dataset_t *ds; char *name, *atp; - char dsname[MAXNAMELEN]; + char dsname[ZFS_MAX_DATASET_NAME_LEN]; name = nvpair_name(pair); atp = strchr(name, '@'); @@ -1460,7 +1494,7 @@ dsl_dataset_snapshot(nvlist_t *snaps, nvlist_t *props, nvlist_t *errors) suspended = fnvlist_alloc(); for (pair = nvlist_next_nvpair(snaps, NULL); pair != NULL; pair = nvlist_next_nvpair(snaps, pair)) { - char fsname[MAXNAMELEN]; + char fsname[ZFS_MAX_DATASET_NAME_LEN]; char *snapname = nvpair_name(pair); char *atp; void *cookie; @@ -1614,6 +1648,21 @@ dsl_dataset_sync(dsl_dataset_t *ds, zio_t *zio, dmu_tx_t *tx) dmu_buf_will_dirty(ds->ds_dbuf, tx); dsl_dataset_phys(ds)->ds_fsid_guid = ds->ds_fsid_guid; + if (ds->ds_resume_bytes[tx->tx_txg & TXG_MASK] != 0) { + VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, + ds->ds_object, DS_FIELD_RESUME_OBJECT, 8, 1, + &ds->ds_resume_object[tx->tx_txg & TXG_MASK], tx)); + VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, + ds->ds_object, DS_FIELD_RESUME_OFFSET, 8, 1, + &ds->ds_resume_offset[tx->tx_txg & TXG_MASK], tx)); + VERIFY0(zap_update(tx->tx_pool->dp_meta_objset, + ds->ds_object, DS_FIELD_RESUME_BYTES, 8, 1, + &ds->ds_resume_bytes[tx->tx_txg & TXG_MASK], tx)); + ds->ds_resume_object[tx->tx_txg & TXG_MASK] = 0; + ds->ds_resume_offset[tx->tx_txg & TXG_MASK] = 0; + ds->ds_resume_bytes[tx->tx_txg & TXG_MASK] = 0; + } + dmu_objset_sync(ds->ds_objset, zio, tx); for (f = 0; f < SPA_FEATURES; f++) { @@ -1654,7 +1703,7 @@ get_clones_stat(dsl_dataset_t *ds, nvlist_t *nv) zap_cursor_retrieve(&zc, &za) == 0; zap_cursor_advance(&zc)) { dsl_dataset_t *clone; - char buf[ZFS_MAXNAMELEN]; + char buf[ZFS_MAX_DATASET_NAME_LEN]; VERIFY0(dsl_dataset_hold_obj(ds->ds_dir->dd_pool, za.za_first_integer, FTAG, &clone)); dsl_dir_name(clone->ds_dir, buf); @@ -1669,6 +1718,78 @@ get_clones_stat(dsl_dataset_t *ds, nvlist_t *nv) nvlist_free(propval); } +static void +get_receive_resume_stats(dsl_dataset_t *ds, nvlist_t *nv) +{ + dsl_pool_t *dp = ds->ds_dir->dd_pool; + + if (dsl_dataset_has_resume_receive_state(ds)) { + char *str; + void *packed; + uint8_t *compressed; + uint64_t val; + nvlist_t *token_nv = fnvlist_alloc(); + size_t packed_size, compressed_size; + zio_cksum_t cksum; + char *propval; + char buf[MAXNAMELEN]; + int i; + + if (zap_lookup(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val) == 0) { + fnvlist_add_uint64(token_nv, "fromguid", val); + } + if (zap_lookup(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val) == 0) { + fnvlist_add_uint64(token_nv, "object", val); + } + if (zap_lookup(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val) == 0) { + fnvlist_add_uint64(token_nv, "offset", val); + } + if (zap_lookup(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_BYTES, sizeof (val), 1, &val) == 0) { + fnvlist_add_uint64(token_nv, "bytes", val); + } + if (zap_lookup(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val) == 0) { + fnvlist_add_uint64(token_nv, "toguid", val); + } + if (zap_lookup(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_TONAME, 1, sizeof (buf), buf) == 0) { + fnvlist_add_string(token_nv, "toname", buf); + } + if (zap_contains(dp->dp_meta_objset, ds->ds_object, + DS_FIELD_RESUME_EMBEDOK) == 0) { + fnvlist_add_boolean(token_nv, "embedok"); + } + packed = fnvlist_pack(token_nv, &packed_size); + fnvlist_free(token_nv); + compressed = kmem_alloc(packed_size, KM_SLEEP); + + compressed_size = gzip_compress(packed, compressed, + packed_size, packed_size, 6); + + fletcher_4_native(compressed, compressed_size, &cksum); + + str = kmem_alloc(compressed_size * 2 + 1, KM_SLEEP); + for (i = 0; i < compressed_size; i++) { + (void) sprintf(str + i * 2, "%02x", compressed[i]); + } + str[compressed_size * 2] = '\0'; + propval = kmem_asprintf("%u-%llx-%llx-%s", + ZFS_SEND_RESUME_TOKEN_VERSION, + (longlong_t)cksum.zc_word[0], + (longlong_t)packed_size, str); + dsl_prop_nvlist_add_string(nv, + ZFS_PROP_RECEIVE_RESUME_TOKEN, propval); + kmem_free(packed, packed_size); + kmem_free(str, compressed_size * 2 + 1); + kmem_free(compressed, packed_size); + strfree(propval); + } +} + void dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv) { @@ -1692,7 +1813,7 @@ dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv) get_clones_stat(ds, nv); } else { if (ds->ds_prev != NULL && ds->ds_prev != dp->dp_origin_snap) { - char buf[MAXNAMELEN]; + char buf[ZFS_MAX_DATASET_NAME_LEN]; dsl_dataset_name(ds->ds_prev, buf); dsl_prop_nvlist_add_string(nv, ZFS_PROP_PREV_SNAP, buf); } @@ -1742,6 +1863,32 @@ dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv) } } + if (!dsl_dataset_is_snapshot(ds)) { + /* 6 extra bytes for /%recv */ + char recvname[ZFS_MAX_DATASET_NAME_LEN + 6]; + dsl_dataset_t *recv_ds; + + /* + * A failed "newfs" (e.g. full) resumable receive leaves + * the stats set on this dataset. Check here for the prop. + */ + get_receive_resume_stats(ds, nv); + + /* + * A failed incremental resumable receive leaves the + * stats set on our child named "%recv". Check the child + * for the prop. + */ + dsl_dataset_name(ds, recvname); + if (strlcat(recvname, "/", sizeof (recvname)) < + sizeof (recvname) && + strlcat(recvname, recv_clone_name, sizeof (recvname)) < + sizeof (recvname) && + dsl_dataset_hold(dp, recvname, FTAG, &recv_ds) == 0) { + get_receive_resume_stats(recv_ds, nv); + dsl_dataset_rele(recv_ds, FTAG); + } + } } void @@ -1862,7 +2009,7 @@ dsl_dataset_rename_snapshot_check_impl(dsl_pool_t *dp, /* dataset name + 1 for the "@" + the new snapshot name must fit */ if (dsl_dir_namelen(hds->ds_dir) + 1 + - strlen(ddrsa->ddrsa_newsnapname) >= MAXNAMELEN) + strlen(ddrsa->ddrsa_newsnapname) >= ZFS_MAX_DATASET_NAME_LEN) error = SET_ERROR(ENAMETOOLONG); return (error); @@ -1969,7 +2116,8 @@ dsl_dataset_rename_snapshot(const char *fsname, * only one long hold on the dataset. We're not allowed to change anything here * so we don't permanently release the long hold or regular hold here. We want * to do this only when syncing to avoid the dataset unexpectedly going away - * when we release the long hold. + * when we release the long hold. Allow a long hold to exist for volumes, this + * may occur when asynchronously registering the minor with the kernel. */ static int dsl_dataset_handoff_check(dsl_dataset_t *ds, void *owner, dmu_tx_t *tx) @@ -1984,7 +2132,7 @@ dsl_dataset_handoff_check(dsl_dataset_t *ds, void *owner, dmu_tx_t *tx) dsl_dataset_long_rele(ds, owner); } - held = dsl_dataset_long_held(ds); + held = (dsl_dataset_long_held(ds) && (ds->ds_owner != zvol_tag)); if (owner != NULL) dsl_dataset_long_hold(ds, owner); @@ -2094,7 +2242,7 @@ dsl_dataset_rollback_sync(void *arg, dmu_tx_t *tx) dsl_pool_t *dp = dmu_tx_pool(tx); dsl_dataset_t *ds, *clone; uint64_t cloneobj; - char namebuf[ZFS_MAXNAMELEN]; + char namebuf[ZFS_MAX_DATASET_NAME_LEN]; VERIFY0(dsl_dataset_hold(dp, ddra->ddra_fsname, FTAG, &ds)); @@ -2647,7 +2795,7 @@ promote_rele(dsl_dataset_promote_arg_t *ddpa, void *tag) * Promote a clone. * * If it fails due to a conflicting snapshot name, "conflsnap" will be filled - * in with the name. (It must be at least MAXNAMELEN bytes long.) + * in with the name. (It must be at least ZFS_MAX_DATASET_NAME_LEN bytes long.) */ int dsl_dataset_promote(const char *name, char *conflsnap) @@ -2684,6 +2832,12 @@ int dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone, dsl_dataset_t *origin_head, boolean_t force, void *owner, dmu_tx_t *tx) { + /* + * "slack" factor for received datasets with refquota set on them. + * See the bottom of this function for details on its use. + */ + uint64_t refquota_slack = (uint64_t)DMU_MAX_ACCESS * + spa_asize_inflation; int64_t unused_refres_delta; /* they should both be heads */ @@ -2726,10 +2880,22 @@ dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone, dsl_dir_space_available(origin_head->ds_dir, NULL, 0, TRUE)) return (SET_ERROR(ENOSPC)); - /* clone can't be over the head's refquota */ + /* + * The clone can't be too much over the head's refquota. + * + * To ensure that the entire refquota can be used, we allow one + * transaction to exceed the the refquota. Therefore, this check + * needs to also allow for the space referenced to be more than the + * refquota. The maximum amount of space that one transaction can use + * on disk is DMU_MAX_ACCESS * spa_asize_inflation. Allowing this + * overage ensures that we are able to receive a filesystem that + * exceeds the refquota on the source system. + * + * So that overage is the refquota_slack we use below. + */ if (origin_head->ds_quota != 0 && dsl_dataset_phys(clone)->ds_referenced_bytes > - origin_head->ds_quota) + origin_head->ds_quota + refquota_slack) return (SET_ERROR(EDQUOT)); return (0); @@ -2744,8 +2910,13 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone, int64_t unused_refres_delta; ASSERT(clone->ds_reserved == 0); + /* + * NOTE: On DEBUG kernels there could be a race between this and + * the check function if spa_asize_inflation is adjusted... + */ ASSERT(origin_head->ds_quota == 0 || - dsl_dataset_phys(clone)->ds_unique_bytes <= origin_head->ds_quota); + dsl_dataset_phys(clone)->ds_unique_bytes <= origin_head->ds_quota + + DMU_MAX_ACCESS * spa_asize_inflation); ASSERT3P(clone->ds_prev, ==, origin_head->ds_prev); /* @@ -3390,6 +3561,23 @@ dsl_dataset_zapify(dsl_dataset_t *ds, dmu_tx_t *tx) dmu_object_zapify(mos, ds->ds_object, DMU_OT_DSL_DATASET, tx); } +boolean_t +dsl_dataset_is_zapified(dsl_dataset_t *ds) +{ + dmu_object_info_t doi; + + dmu_object_info_from_db(ds->ds_dbuf, &doi); + return (doi.doi_type == DMU_OTN_ZAP_METADATA); +} + +boolean_t +dsl_dataset_has_resume_receive_state(dsl_dataset_t *ds) +{ + return (dsl_dataset_is_zapified(ds) && + zap_contains(ds->ds_dir->dd_pool->dp_meta_objset, + ds->ds_object, DS_FIELD_RESUME_TOGUID) == 0); +} + #if defined(_KERNEL) && defined(HAVE_SPL) #if defined(_LP64) module_param(zfs_max_recordsize, int, 0644); diff --git a/module/zfs/dsl_deleg.c b/module/zfs/dsl_deleg.c index 952422be2381..eb39cff57f2a 100644 --- a/module/zfs/dsl_deleg.c +++ b/module/zfs/dsl_deleg.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2014 by Delphix. All rights reserved. + * Copyright (c) 2011, 2015 by Delphix. All rights reserved. */ /* @@ -330,7 +330,7 @@ dsl_deleg_get(const char *ddname, nvlist_t **nvp) za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); basezc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP); baseza = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); - source = kmem_alloc(MAXNAMELEN + strlen(MOS_DIR_NAME) + 1, KM_SLEEP); + source = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0); for (dd = startdd; dd != NULL; dd = dd->dd_parent) { @@ -370,7 +370,7 @@ dsl_deleg_get(const char *ddname, nvlist_t **nvp) nvlist_free(sp_nvp); } - kmem_free(source, MAXNAMELEN + strlen(MOS_DIR_NAME) + 1); + kmem_free(source, ZFS_MAX_DATASET_NAME_LEN); kmem_free(baseza, sizeof (zap_attribute_t)); kmem_free(basezc, sizeof (zap_cursor_t)); kmem_free(za, sizeof (zap_attribute_t)); diff --git a/module/zfs/dsl_destroy.c b/module/zfs/dsl_destroy.c index d7c34c9a403e..716081ba3ac3 100644 --- a/module/zfs/dsl_destroy.c +++ b/module/zfs/dsl_destroy.c @@ -978,9 +978,17 @@ dsl_destroy_inconsistent(const char *dsname, void *arg) objset_t *os; if (dmu_objset_hold(dsname, FTAG, &os) == 0) { - boolean_t inconsistent = DS_IS_INCONSISTENT(dmu_objset_ds(os)); + boolean_t need_destroy = DS_IS_INCONSISTENT(dmu_objset_ds(os)); + + /* + * If the dataset is inconsistent because a resumable receive + * has failed, then do not destroy it. + */ + if (dsl_dataset_has_resume_receive_state(dmu_objset_ds(os))) + need_destroy = B_FALSE; + dmu_objset_rele(os, FTAG); - if (inconsistent) + if (need_destroy) (void) dsl_destroy_head(dsname); } return (0); diff --git a/module/zfs/dsl_dir.c b/module/zfs/dsl_dir.c index 8983e0793f23..ae67b362ee8a 100644 --- a/module/zfs/dsl_dir.c +++ b/module/zfs/dsl_dir.c @@ -299,13 +299,14 @@ dsl_dir_async_rele(dsl_dir_t *dd, void *tag) dmu_buf_rele(dd->dd_dbuf, tag); } -/* buf must be long enough (MAXNAMELEN + strlen(MOS_DIR_NAME) + 1 should do) */ +/* buf must be at least ZFS_MAX_DATASET_NAME_LEN bytes */ void dsl_dir_name(dsl_dir_t *dd, char *buf) { if (dd->dd_parent) { dsl_dir_name(dd->dd_parent, buf); - (void) strcat(buf, "/"); + VERIFY3U(strlcat(buf, "/", ZFS_MAX_DATASET_NAME_LEN), <, + ZFS_MAX_DATASET_NAME_LEN); } else { buf[0] = '\0'; } @@ -315,10 +316,12 @@ dsl_dir_name(dsl_dir_t *dd, char *buf) * dprintf_dd() with dd_lock held */ mutex_enter(&dd->dd_lock); - (void) strcat(buf, dd->dd_myname); + VERIFY3U(strlcat(buf, dd->dd_myname, ZFS_MAX_DATASET_NAME_LEN), + <, ZFS_MAX_DATASET_NAME_LEN); mutex_exit(&dd->dd_lock); } else { - (void) strcat(buf, dd->dd_myname); + VERIFY3U(strlcat(buf, dd->dd_myname, ZFS_MAX_DATASET_NAME_LEN), + <, ZFS_MAX_DATASET_NAME_LEN); } } @@ -367,12 +370,12 @@ getcomponent(const char *path, char *component, const char **nextp) if (p != NULL && (p[0] != '@' || strpbrk(path+1, "/@") || p[1] == '\0')) return (SET_ERROR(EINVAL)); - if (strlen(path) >= MAXNAMELEN) + if (strlen(path) >= ZFS_MAX_DATASET_NAME_LEN) return (SET_ERROR(ENAMETOOLONG)); (void) strcpy(component, path); p = NULL; } else if (p[0] == '/') { - if (p - path >= MAXNAMELEN) + if (p - path >= ZFS_MAX_DATASET_NAME_LEN) return (SET_ERROR(ENAMETOOLONG)); (void) strncpy(component, path, p - path); component[p - path] = '\0'; @@ -384,7 +387,7 @@ getcomponent(const char *path, char *component, const char **nextp) */ if (strchr(path, '/')) return (SET_ERROR(EINVAL)); - if (p - path >= MAXNAMELEN) + if (p - path >= ZFS_MAX_DATASET_NAME_LEN) return (SET_ERROR(ENAMETOOLONG)); (void) strncpy(component, path, p - path); component[p - path] = '\0'; @@ -412,7 +415,7 @@ dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag, dsl_dir_t *dd; uint64_t ddobj; - buf = kmem_alloc(MAXNAMELEN, KM_SLEEP); + buf = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); err = getcomponent(name, buf, &next); if (err != 0) goto error; @@ -479,7 +482,7 @@ dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag, *tailp = next; *ddp = dd; error: - kmem_free(buf, MAXNAMELEN); + kmem_free(buf, ZFS_MAX_DATASET_NAME_LEN); return (err); } @@ -974,7 +977,7 @@ dsl_dir_stats(dsl_dir_t *dd, nvlist_t *nv) if (dsl_dir_is_clone(dd)) { dsl_dataset_t *ds; - char buf[MAXNAMELEN]; + char buf[ZFS_MAX_DATASET_NAME_LEN]; VERIFY0(dsl_dataset_hold_obj(dd->dd_pool, dsl_dir_phys(dd)->dd_origin_obj, FTAG, &ds)); @@ -1691,11 +1694,11 @@ static int dsl_valid_rename(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) { int *deltap = arg; - char namebuf[MAXNAMELEN]; + char namebuf[ZFS_MAX_DATASET_NAME_LEN]; dsl_dataset_name(ds, namebuf); - if (strlen(namebuf) + *deltap >= MAXNAMELEN) + if (strlen(namebuf) + *deltap >= ZFS_MAX_DATASET_NAME_LEN) return (SET_ERROR(ENAMETOOLONG)); return (0); } diff --git a/module/zfs/dsl_pool.c b/module/zfs/dsl_pool.c old mode 100644 new mode 100755 index ada0eac63eea..cf5259acd5b4 --- a/module/zfs/dsl_pool.c +++ b/module/zfs/dsl_pool.c @@ -182,12 +182,20 @@ dsl_pool_init(spa_t *spa, uint64_t txg, dsl_pool_t **dpp) int err; dsl_pool_t *dp = dsl_pool_open_impl(spa, txg); + /* + * Initialize the caller's dsl_pool_t structure before we actually open + * the meta objset. This is done because a self-healing write zio may + * be issued as part of dmu_objset_open_impl() and the spa needs its + * dsl_pool_t initialized in order to handle the write. + */ + *dpp = dp; + err = dmu_objset_open_impl(spa, NULL, &dp->dp_meta_rootbp, &dp->dp_meta_objset); - if (err != 0) + if (err != 0) { dsl_pool_close(dp); - else - *dpp = dp; + *dpp = NULL; + } return (err); } diff --git a/module/zfs/dsl_prop.c b/module/zfs/dsl_prop.c index 36147327550d..66e899a57118 100644 --- a/module/zfs/dsl_prop.c +++ b/module/zfs/dsl_prop.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. * Copyright (c) 2013 Martin Matuska. All rights reserved. * Copyright 2015, Joyent, Inc. */ @@ -1095,7 +1095,7 @@ dsl_prop_get_all_ds(dsl_dataset_t *ds, nvlist_t **nvp, dsl_pool_t *dp = dd->dd_pool; objset_t *mos = dp->dp_meta_objset; int err = 0; - char setpoint[MAXNAMELEN]; + char setpoint[ZFS_MAX_DATASET_NAME_LEN]; VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0); diff --git a/module/zfs/dsl_scan.c b/module/zfs/dsl_scan.c index 8a5fac25f498..6c5f1f0b5b3f 100644 --- a/module/zfs/dsl_scan.c +++ b/module/zfs/dsl_scan.c @@ -56,7 +56,8 @@ typedef int (scan_cb_t)(dsl_pool_t *, const blkptr_t *, static scan_cb_t dsl_scan_scrub_cb; static void dsl_scan_cancel_sync(void *, dmu_tx_t *); -static void dsl_scan_sync_state(dsl_scan_t *, dmu_tx_t *tx); +static void dsl_scan_sync_state(dsl_scan_t *, dmu_tx_t *); +static boolean_t dsl_scan_restarting(dsl_scan_t *, dmu_tx_t *); int zfs_top_maxinflight = 32; /* maximum I/Os per top-level */ int zfs_resilver_delay = 2; /* number of ticks to delay resilver */ @@ -329,8 +330,15 @@ dsl_scan_done(dsl_scan_t *scn, boolean_t complete, dmu_tx_t *tx) else scn->scn_phys.scn_state = DSS_CANCELED; - spa_history_log_internal(spa, "scan done", tx, - "complete=%u", complete); + if (dsl_scan_restarting(scn, tx)) + spa_history_log_internal(spa, "scan aborted, restarting", tx, + "errors=%llu", spa_get_errlog_size(spa)); + else if (!complete) + spa_history_log_internal(spa, "scan cancelled", tx, + "errors=%llu", spa_get_errlog_size(spa)); + else + spa_history_log_internal(spa, "scan done", tx, + "errors=%llu", spa_get_errlog_size(spa)); if (DSL_SCAN_IS_SCRUB_RESILVER(scn)) { mutex_enter(&spa->spa_scrub_lock); @@ -701,14 +709,18 @@ dsl_scan_recurse(dsl_scan_t *scn, dsl_dataset_t *ds, dmu_objset_type_t ostype, scn->scn_phys.scn_errors++; return (err); } - for (i = 0, cdnp = buf->b_data; i < epb; i++, cdnp++) { + for (i = 0, cdnp = buf->b_data; i < epb; + i += cdnp->dn_extra_slots + 1, + cdnp += cdnp->dn_extra_slots + 1) { for (j = 0; j < cdnp->dn_nblkptr; j++) { blkptr_t *cbp = &cdnp->dn_blkptr[j]; dsl_scan_prefetch(scn, buf, cbp, zb->zb_objset, zb->zb_blkid * epb + i, j); } } - for (i = 0, cdnp = buf->b_data; i < epb; i++, cdnp++) { + for (i = 0, cdnp = buf->b_data; i < epb; + i += cdnp->dn_extra_slots + 1, + cdnp += cdnp->dn_extra_slots + 1) { dsl_scan_visitdnode(scn, ds, ostype, cdnp, zb->zb_blkid * epb + i, tx); } @@ -771,7 +783,7 @@ dsl_scan_visitdnode(dsl_scan_t *scn, dsl_dataset_t *ds, zbookmark_phys_t czb; SET_BOOKMARK(&czb, ds ? ds->ds_object : 0, object, 0, DMU_SPILL_BLKID); - dsl_scan_visitbp(&dnp->dn_spill, + dsl_scan_visitbp(DN_SPILL_BLKPTR(dnp), &czb, dnp, ds, scn, ostype, tx); } } @@ -1103,7 +1115,7 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) * rootbp's birth time is < cur_min_txg. Then we will * add the next snapshots/clones to the work queue. */ - char *dsname = kmem_alloc(MAXNAMELEN, KM_SLEEP); + char *dsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); dsl_dataset_name(ds, dsname); zfs_dbgmsg("scanning dataset %llu (%s) is unnecessary because " "cur_min_txg (%llu) >= max_txg (%llu)", @@ -1134,7 +1146,7 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) dmu_buf_will_dirty(ds->ds_dbuf, tx); dsl_scan_visit_rootbp(scn, ds, &dsl_dataset_phys(ds)->ds_bp, tx); - dsname = kmem_alloc(ZFS_MAXNAMELEN, KM_SLEEP); + dsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); dsl_dataset_name(ds, dsname); zfs_dbgmsg("scanned dataset %llu (%s) with min=%llu max=%llu; " "pausing=%u", @@ -1142,7 +1154,7 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx) (longlong_t)scn->scn_phys.scn_cur_min_txg, (longlong_t)scn->scn_phys.scn_cur_max_txg, (int)scn->scn_pausing); - kmem_free(dsname, ZFS_MAXNAMELEN); + kmem_free(dsname, ZFS_MAX_DATASET_NAME_LEN); if (scn->scn_pausing) goto out; @@ -1511,8 +1523,7 @@ dsl_scan_sync(dsl_pool_t *dp, dmu_tx_t *tx) * that we can restart an old-style scan while the pool is being * imported (see dsl_scan_init). */ - if (scn->scn_restart_txg != 0 && - scn->scn_restart_txg <= tx->tx_txg) { + if (dsl_scan_restarting(scn, tx)) { pool_scan_func_t func = POOL_SCAN_SCRUB; dsl_scan_done(scn, B_FALSE, tx); if (vdev_resilver_needed(spa->spa_root_vdev, NULL, NULL)) @@ -1942,6 +1953,13 @@ dsl_scan(dsl_pool_t *dp, pool_scan_func_t func) dsl_scan_setup_sync, &func, 0, ZFS_SPACE_CHECK_NONE)); } +static boolean_t +dsl_scan_restarting(dsl_scan_t *scn, dmu_tx_t *tx) +{ + return (scn->scn_restart_txg != 0 && + scn->scn_restart_txg <= tx->tx_txg); +} + #if defined(_KERNEL) && defined(HAVE_SPL) module_param(zfs_top_maxinflight, int, 0644); MODULE_PARM_DESC(zfs_top_maxinflight, "Max I/Os per top-level"); diff --git a/module/zfs/dsl_userhold.c b/module/zfs/dsl_userhold.c index 1b234ed480f9..a6d1aa937ef1 100644 --- a/module/zfs/dsl_userhold.c +++ b/module/zfs/dsl_userhold.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. * Copyright (c) 2013 Steven Hartland. All rights reserved. */ @@ -181,7 +181,7 @@ dsl_dataset_user_hold_sync_one_impl(nvlist_t *tmpholds, dsl_dataset_t *ds, } typedef struct zfs_hold_cleanup_arg { - char zhca_spaname[MAXNAMELEN]; + char zhca_spaname[ZFS_MAX_DATASET_NAME_LEN]; uint64_t zhca_spa_load_guid; nvlist_t *zhca_holds; } zfs_hold_cleanup_arg_t; @@ -580,7 +580,7 @@ dsl_dataset_user_release_impl(nvlist_t *holds, nvlist_t *errlist, error = dsl_dataset_hold_obj_string(tmpdp, nvpair_name(pair), FTAG, &ds); if (error == 0) { - char name[MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; dsl_dataset_name(ds, name); dsl_pool_config_exit(tmpdp, FTAG); dsl_dataset_rele(ds, FTAG); diff --git a/module/zfs/policy.c b/module/zfs/policy.c new file mode 100644 index 000000000000..fda13a9b52b5 --- /dev/null +++ b/module/zfs/policy.c @@ -0,0 +1,303 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright 2013, Joyent, Inc. All rights reserved. + * Copyright (C) 2016 Lawrence Livermore National Security, LLC. + * + * For Linux the vast majority of this enforcement is already handled via + * the standard Linux VFS permission checks. However certain administrative + * commands which bypass the standard mechanisms may need to make use of + * this functionality. + */ + +#include +#include +#include + +/* + * The passed credentials cannot be directly verified because Linux only + * provides and interface to check the *current* proces credentials. In + * order to handle this the capable() test is only run when the passed + * credentials match the current process credentials or the kcred. In + * all other cases this function must fail and return the passed err. + */ +static int +priv_policy(const cred_t *cr, int capability, boolean_t all, int err) +{ + ASSERT3S(all, ==, B_FALSE); + + if (cr != CRED() && (cr != kcred)) + return (err); + + if (!capable(capability)) + return (err); + + return (0); +} + +/* + * Checks for operations that are either client-only or are used by + * both clients and servers. + */ +int +secpolicy_nfs(const cred_t *cr) +{ + return (priv_policy(cr, CAP_SYS_ADMIN, B_FALSE, EPERM)); +} + +/* + * Catch all system configuration. + */ +int +secpolicy_sys_config(const cred_t *cr, boolean_t checkonly) +{ + return (priv_policy(cr, CAP_SYS_ADMIN, B_FALSE, EPERM)); +} + +/* + * Like secpolicy_vnode_access() but we get the actual wanted mode and the + * current mode of the file, not the missing bits. + * + * Enforced in the Linux VFS. + */ +int +secpolicy_vnode_access2(const cred_t *cr, struct inode *ip, uid_t owner, + mode_t curmode, mode_t wantmode) +{ + return (0); +} + +/* + * This is a special routine for ZFS; it is used to determine whether + * any of the privileges in effect allow any form of access to the + * file. There's no reason to audit this or any reason to record + * this. More work is needed to do the "KPLD" stuff. + */ +int +secpolicy_vnode_any_access(const cred_t *cr, struct inode *ip, uid_t owner) +{ + if (crgetfsuid(cr) == owner) + return (0); + + if (zpl_inode_owner_or_capable(ip)) + return (0); + + if (priv_policy(cr, CAP_DAC_OVERRIDE, B_FALSE, EPERM) == 0) + return (0); + + if (priv_policy(cr, CAP_DAC_READ_SEARCH, B_FALSE, EPERM) == 0) + return (0); + + return (EPERM); +} + +/* + * Determine if subject can chown owner of a file. + */ +int +secpolicy_vnode_chown(const cred_t *cr, uid_t owner) +{ + if (crgetfsuid(cr) == owner) + return (0); + + return (priv_policy(cr, CAP_FOWNER, B_FALSE, EPERM)); +} + +/* + * Determine if subject can change group ownership of a file. + */ +int +secpolicy_vnode_create_gid(const cred_t *cr) +{ + return (priv_policy(cr, CAP_SETGID, B_FALSE, EPERM)); +} + +/* + * Policy determines whether we can remove an entry from a directory, + * regardless of permission bits. + */ +int +secpolicy_vnode_remove(const cred_t *cr) +{ + return (priv_policy(cr, CAP_FOWNER, B_FALSE, EPERM)); +} + +/* + * Determine that subject can modify the mode of a file. allzone privilege + * needed when modifying root owned object. + */ +int +secpolicy_vnode_setdac(const cred_t *cr, uid_t owner) +{ + if (crgetfsuid(cr) == owner) + return (0); + + return (priv_policy(cr, CAP_FOWNER, B_FALSE, EPERM)); +} + +/* + * Are we allowed to retain the set-uid/set-gid bits when + * changing ownership or when writing to a file? + * "issuid" should be true when set-uid; only in that case + * root ownership is checked (setgid is assumed). + * + * Enforced in the Linux VFS. + */ +int +secpolicy_vnode_setid_retain(const cred_t *cr, boolean_t issuidroot) +{ + return (0); +} + +/* + * Determine that subject can set the file setgid flag. + */ +int +secpolicy_vnode_setids_setgids(const cred_t *cr, gid_t gid) +{ + if (crgetfsgid(cr) != gid && !groupmember(gid, cr)) + return (priv_policy(cr, CAP_FSETID, B_FALSE, EPERM)); + + return (0); +} + +/* + * Determine if the subject can inject faults in the ZFS fault injection + * framework. Requires all privileges. + */ +int +secpolicy_zinject(const cred_t *cr) +{ + return (priv_policy(cr, CAP_SYS_ADMIN, B_FALSE, EACCES)); +} + +/* + * Determine if the subject has permission to manipulate ZFS datasets + * (not pools). Equivalent to the SYS_MOUNT privilege. + */ +int +secpolicy_zfs(const cred_t *cr) +{ + return (priv_policy(cr, CAP_SYS_ADMIN, B_FALSE, EACCES)); +} + +void +secpolicy_setid_clear(vattr_t *vap, cred_t *cr) +{ + if ((vap->va_mode & (S_ISUID | S_ISGID)) != 0 && + secpolicy_vnode_setid_retain(cr, + (vap->va_mode & S_ISUID) != 0 && + (vap->va_mask & AT_UID) != 0 && vap->va_uid == 0) != 0) { + vap->va_mask |= AT_MODE; + vap->va_mode &= ~(S_ISUID|S_ISGID); + } +} + +/* + * Determine that subject can set the file setid flags. + */ +static int +secpolicy_vnode_setid_modify(const cred_t *cr, uid_t owner) +{ + if (crgetfsuid(cr) == owner) + return (0); + + return (priv_policy(cr, CAP_FSETID, B_FALSE, EPERM)); +} + +/* + * Determine that subject can make a file a "sticky". + * + * Enforced in the Linux VFS. + */ +static int +secpolicy_vnode_stky_modify(const cred_t *cr) +{ + return (0); +} + +int +secpolicy_setid_setsticky_clear(struct inode *ip, vattr_t *vap, + const vattr_t *ovap, cred_t *cr) +{ + int error; + + if ((vap->va_mode & S_ISUID) != 0 && + (error = secpolicy_vnode_setid_modify(cr, + ovap->va_uid)) != 0) { + return (error); + } + + /* + * Check privilege if attempting to set the + * sticky bit on a non-directory. + */ + if (!S_ISDIR(ip->i_mode) && (vap->va_mode & S_ISVTX) != 0 && + secpolicy_vnode_stky_modify(cr) != 0) { + vap->va_mode &= ~S_ISVTX; + } + + /* + * Check for privilege if attempting to set the + * group-id bit. + */ + if ((vap->va_mode & S_ISGID) != 0 && + secpolicy_vnode_setids_setgids(cr, ovap->va_gid) != 0) { + vap->va_mode &= ~S_ISGID; + } + + return (0); +} + +/* + * Check privileges for setting xvattr attributes + */ +int +secpolicy_xvattr(xvattr_t *xvap, uid_t owner, cred_t *cr, vtype_t vtype) +{ + return (secpolicy_vnode_chown(cr, owner)); +} + +/* + * Check privileges for setattr attributes. + * + * Enforced in the Linux VFS. + */ +int +secpolicy_vnode_setattr(cred_t *cr, struct inode *ip, struct vattr *vap, + const struct vattr *ovap, int flags, + int unlocked_access(void *, int, cred_t *), void *node) +{ + return (0); +} + +/* + * Check privileges for links. + * + * Enforced in the Linux VFS. + */ +int +secpolicy_basic_link(const cred_t *cr) +{ + return (0); +} diff --git a/module/zfs/sa.c b/module/zfs/sa.c index d6ac5fcc709a..adc301512d31 100644 --- a/module/zfs/sa.c +++ b/module/zfs/sa.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -553,12 +554,11 @@ sa_copy_data(sa_data_locator_t *func, void *datastart, void *target, int buflen) */ static int sa_find_sizes(sa_os_t *sa, sa_bulk_attr_t *attr_desc, int attr_count, - dmu_buf_t *db, sa_buf_type_t buftype, int *index, int *total, - boolean_t *will_spill) + dmu_buf_t *db, sa_buf_type_t buftype, int full_space, int *index, + int *total, boolean_t *will_spill) { int var_size_count = 0; int i; - int full_space; int hdrsize; int extra_hdrsize; @@ -577,7 +577,6 @@ sa_find_sizes(sa_os_t *sa, sa_bulk_attr_t *attr_desc, int attr_count, hdrsize = (SA_BONUSTYPE_FROM_DB(db) == DMU_OT_ZNODE) ? 0 : sizeof (sa_hdr_phys_t); - full_space = (buftype == SA_BONUS) ? DN_MAX_BONUSLEN : db->db_size; ASSERT(IS_P2ALIGNED(full_space, 8)); for (i = 0; i != attr_count; i++) { @@ -668,6 +667,7 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count, void *data_start; sa_attr_type_t *attrs, *attrs_start; int i, lot_count; + int dnodesize; int spill_idx; int hdrsize; int spillhdrsize = 0; @@ -676,20 +676,23 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count, sa_lot_t *lot; int len_idx; int spill_used; + int bonuslen; boolean_t spilling; dmu_buf_will_dirty(hdl->sa_bonus, tx); bonustype = SA_BONUSTYPE_FROM_DB(hdl->sa_bonus); + dmu_object_dnsize_from_db(hdl->sa_bonus, &dnodesize); + bonuslen = DN_BONUS_SIZE(dnodesize); /* first determine bonus header size and sum of all attributes */ hdrsize = sa_find_sizes(sa, attr_desc, attr_count, hdl->sa_bonus, - SA_BONUS, &spill_idx, &used, &spilling); + SA_BONUS, bonuslen, &spill_idx, &used, &spilling); if (used > SPA_OLD_MAXBLOCKSIZE) return (SET_ERROR(EFBIG)); - VERIFY(0 == dmu_set_bonus(hdl->sa_bonus, spilling ? - MIN(DN_MAX_BONUSLEN - sizeof (blkptr_t), used + hdrsize) : + VERIFY0(dmu_set_bonus(hdl->sa_bonus, spilling ? + MIN(bonuslen - sizeof (blkptr_t), used + hdrsize) : used + hdrsize, tx)); ASSERT((bonustype == DMU_OT_ZNODE && spilling == 0) || @@ -706,8 +709,8 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count, dmu_buf_will_dirty(hdl->sa_spill, tx); spillhdrsize = sa_find_sizes(sa, &attr_desc[spill_idx], - attr_count - spill_idx, hdl->sa_spill, SA_SPILL, &i, - &spill_used, &dummy); + attr_count - spill_idx, hdl->sa_spill, SA_SPILL, + hdl->sa_spill->db_size, &i, &spill_used, &dummy); if (spill_used > SPA_OLD_MAXBLOCKSIZE) return (SET_ERROR(EFBIG)); diff --git a/module/zfs/spa.c b/module/zfs/spa.c index c23fd7a3aad3..15c87866e69e 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -281,6 +281,14 @@ spa_prop_get_config(spa_t *spa, nvlist_t **nvp) SPA_OLD_MAXBLOCKSIZE, ZPROP_SRC_NONE); } + if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) { + spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, + DNODE_MAX_SIZE, ZPROP_SRC_NONE); + } else { + spa_prop_add_list(*nvp, ZPOOL_PROP_MAXDNODESIZE, NULL, + DNODE_MIN_SIZE, ZPROP_SRC_NONE); + } + if ((dp = list_head(&spa->spa_config_list)) != NULL) { if (dp->scd_path == NULL) { spa_prop_add_list(*nvp, ZPOOL_PROP_CACHEFILE, @@ -353,8 +361,7 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) break; } - strval = kmem_alloc( - MAXNAMELEN + strlen(MOS_DIR_NAME) + 1, + strval = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); dsl_dataset_name(ds, strval); dsl_dataset_rele(ds, FTAG); @@ -367,8 +374,7 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) spa_prop_add_list(*nvp, prop, strval, intval, src); if (strval != NULL) - kmem_free(strval, - MAXNAMELEN + strlen(MOS_DIR_NAME) + 1); + kmem_free(strval, ZFS_MAX_DATASET_NAME_LEN); break; @@ -512,7 +518,8 @@ spa_prop_validate(spa_t *spa, nvlist_t *props) /* * Must be ZPL, and its property settings * must be supported by GRUB (compression - * is not gzip, and large blocks are not used). + * is not gzip, and large blocks or large + * dnodes are not used). */ if (dmu_objset_type(os) != DMU_OST_ZFS) { @@ -529,6 +536,12 @@ spa_prop_validate(spa_t *spa, nvlist_t *props) &propval)) == 0 && propval > SPA_OLD_MAXBLOCKSIZE) { error = SET_ERROR(ENOTSUP); + } else if ((error = + dsl_prop_get_int_ds(dmu_objset_ds(os), + zfs_prop_to_name(ZFS_PROP_DNODESIZE), + &propval)) == 0 && + propval != ZFS_DNSIZE_LEGACY) { + error = SET_ERROR(ENOTSUP); } else { objnum = dmu_objset_id(os); } @@ -2003,6 +2016,16 @@ spa_load_verify_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp, return (0); } +/* ARGSUSED */ +int +verify_dataset_name_len(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) +{ + if (dsl_dataset_namelen(ds) >= ZFS_MAX_DATASET_NAME_LEN) + return (SET_ERROR(ENAMETOOLONG)); + + return (0); +} + static int spa_load_verify(spa_t *spa) { @@ -2017,6 +2040,14 @@ spa_load_verify(spa_t *spa) if (policy.zrp_request & ZPOOL_NEVER_REWIND) return (0); + dsl_pool_config_enter(spa->spa_dsl_pool, FTAG); + error = dmu_objset_find_dp(spa->spa_dsl_pool, + spa->spa_dsl_pool->dp_root_dir_obj, verify_dataset_name_len, NULL, + DS_FIND_CHILDREN); + dsl_pool_config_exit(spa->spa_dsl_pool, FTAG); + if (error != 0) + return (error); + rio = zio_root(spa, NULL, &sle, ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE); @@ -3940,7 +3971,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, return (0); } -#ifdef _KERNEL +#if defined(_KERNEL) && !defined(__linux__) /* * Get the root pool information from the root disk, then import the root pool * during the system boot up time. @@ -4143,7 +4174,7 @@ spa_import_rootpool(char *devpath, char *devid) return (error); } -#endif +#endif /* defined(_KERNEL) && !defined(__linux__) */ /* * Import a non-root pool into the system. @@ -7007,7 +7038,6 @@ EXPORT_SYMBOL(spa_open); EXPORT_SYMBOL(spa_open_rewind); EXPORT_SYMBOL(spa_get_stats); EXPORT_SYMBOL(spa_create); -EXPORT_SYMBOL(spa_import_rootpool); EXPORT_SYMBOL(spa_import); EXPORT_SYMBOL(spa_tryimport); EXPORT_SYMBOL(spa_destroy); diff --git a/module/zfs/spa_history.c b/module/zfs/spa_history.c index 01aa4641e63f..cf6fc224a230 100644 --- a/module/zfs/spa_history.c +++ b/module/zfs/spa_history.c @@ -21,7 +21,7 @@ /* * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2014 by Delphix. All rights reserved. + * Copyright (c) 2011, 2015 by Delphix. All rights reserved. */ #include @@ -493,7 +493,7 @@ spa_history_log_internal_ds(dsl_dataset_t *ds, const char *operation, dmu_tx_t *tx, const char *fmt, ...) { va_list adx; - char namebuf[MAXNAMELEN]; + char namebuf[ZFS_MAX_DATASET_NAME_LEN]; nvlist_t *nvl = fnvlist_alloc(); ASSERT(tx != NULL); @@ -512,7 +512,7 @@ spa_history_log_internal_dd(dsl_dir_t *dd, const char *operation, dmu_tx_t *tx, const char *fmt, ...) { va_list adx; - char namebuf[MAXNAMELEN]; + char namebuf[ZFS_MAX_DATASET_NAME_LEN]; nvlist_t *nvl = fnvlist_alloc(); ASSERT(tx != NULL); diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index 6f2ba52f4e3e..d1303b5c2205 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -1831,6 +1832,7 @@ spa_init(int mode) dmu_init(); zil_init(); vdev_cache_stat_init(); + vdev_raidz_math_init(); zfs_prop_init(); zpool_prop_init(); zpool_feature_init(); @@ -1846,6 +1848,7 @@ spa_fini(void) spa_evict_all(); vdev_cache_stat_fini(); + vdev_raidz_math_fini(); zil_fini(); dmu_fini(); zio_fini(); @@ -1997,6 +2000,15 @@ spa_maxblocksize(spa_t *spa) return (SPA_OLD_MAXBLOCKSIZE); } +int +spa_maxdnodesize(spa_t *spa) +{ + if (spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE)) + return (DNODE_MAX_SIZE); + else + return (DNODE_MIN_SIZE); +} + #if defined(_KERNEL) && defined(HAVE_SPL) /* Namespace manipulation */ EXPORT_SYMBOL(spa_lookup); @@ -2053,6 +2065,7 @@ EXPORT_SYMBOL(spa_bootfs); EXPORT_SYMBOL(spa_delegation); EXPORT_SYMBOL(spa_meta_objset); EXPORT_SYMBOL(spa_maxblocksize); +EXPORT_SYMBOL(spa_maxdnodesize); /* Miscellaneous support routines */ EXPORT_SYMBOL(spa_rename); diff --git a/module/zfs/trace.c b/module/zfs/trace.c index 0c9990e8547b..e4ebf31b3fbe 100644 --- a/module/zfs/trace.c +++ b/module/zfs/trace.c @@ -47,4 +47,5 @@ #include #include #include +#include #include diff --git a/module/zfs/txg.c b/module/zfs/txg.c index ed8007e05238..9dda58c283df 100644 --- a/module/zfs/txg.c +++ b/module/zfs/txg.c @@ -365,6 +365,7 @@ static void txg_quiesce(dsl_pool_t *dp, uint64_t txg) { tx_state_t *tx = &dp->dp_tx; + uint64_t tx_open_time; int g = txg & TXG_MASK; int c; @@ -376,10 +377,7 @@ txg_quiesce(dsl_pool_t *dp, uint64_t txg) ASSERT(txg == tx->tx_open_txg); tx->tx_open_txg++; - tx->tx_open_time = gethrtime(); - - spa_txg_history_set(dp->dp_spa, txg, TXG_STATE_OPEN, tx->tx_open_time); - spa_txg_history_add(dp->dp_spa, tx->tx_open_txg, tx->tx_open_time); + tx->tx_open_time = tx_open_time = gethrtime(); DTRACE_PROBE2(txg__quiescing, dsl_pool_t *, dp, uint64_t, txg); DTRACE_PROBE2(txg__opened, dsl_pool_t *, dp, uint64_t, tx->tx_open_txg); @@ -391,6 +389,9 @@ txg_quiesce(dsl_pool_t *dp, uint64_t txg) for (c = 0; c < max_ncpus; c++) mutex_exit(&tx->tx_cpu[c].tc_open_lock); + spa_txg_history_set(dp->dp_spa, txg, TXG_STATE_OPEN, tx_open_time); + spa_txg_history_add(dp->dp_spa, txg + 1, tx_open_time); + /* * Quiesce the transaction group by waiting for everyone to txg_exit(). */ diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index 13739017382a..302458b73b2d 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -704,6 +704,7 @@ vdev_top_transfer(vdev_t *svd, vdev_t *tvd) ASSERT(tvd == tvd->vdev_top); + tvd->vdev_pending_fastwrite = svd->vdev_pending_fastwrite; tvd->vdev_ms_array = svd->vdev_ms_array; tvd->vdev_ms_shift = svd->vdev_ms_shift; tvd->vdev_ms_count = svd->vdev_ms_count; @@ -892,11 +893,11 @@ vdev_metaslab_init(vdev_t *vd, uint64_t txg) ASSERT(oldc <= newc); - mspp = kmem_zalloc(newc * sizeof (*mspp), KM_SLEEP); + mspp = vmem_zalloc(newc * sizeof (*mspp), KM_SLEEP); if (oldc != 0) { bcopy(vd->vdev_ms, mspp, oldc * sizeof (*mspp)); - kmem_free(vd->vdev_ms, oldc * sizeof (*mspp)); + vmem_free(vd->vdev_ms, oldc * sizeof (*mspp)); } vd->vdev_ms = mspp; @@ -950,7 +951,7 @@ vdev_metaslab_fini(vdev_t *vd) if (msp != NULL) metaslab_fini(msp); } - kmem_free(vd->vdev_ms, count * sizeof (metaslab_t *)); + vmem_free(vd->vdev_ms, count * sizeof (metaslab_t *)); vd->vdev_ms = NULL; } @@ -1909,12 +1910,15 @@ vdev_dtl_reassess(vdev_t *vd, uint64_t txg, uint64_t scrub_txg, int scrub_done) /* * If the vdev was resilvering and no longer has any - * DTLs then reset its resilvering flag. + * DTLs then reset its resilvering flag and dirty + * the top level so that we persist the change. */ if (vd->vdev_resilver_txg != 0 && range_tree_space(vd->vdev_dtl[DTL_MISSING]) == 0 && - range_tree_space(vd->vdev_dtl[DTL_OUTAGE]) == 0) + range_tree_space(vd->vdev_dtl[DTL_OUTAGE]) == 0) { vd->vdev_resilver_txg = 0; + vdev_config_dirty(vd->vdev_top); + } mutex_exit(&vd->vdev_dtl_lock); @@ -2784,21 +2788,30 @@ vdev_get_child_stat_ex(vdev_t *cvd, vdev_stat_ex_t *vsx, vdev_stat_ex_t *cvsx) { int t, b; for (t = 0; t < ZIO_TYPES; t++) { - for (b = 0; b < VDEV_HISTO_BUCKETS; b++) { + for (b = 0; b < ARRAY_SIZE(vsx->vsx_disk_histo[0]); b++) vsx->vsx_disk_histo[t][b] += cvsx->vsx_disk_histo[t][b]; + + for (b = 0; b < ARRAY_SIZE(vsx->vsx_total_histo[0]); b++) { vsx->vsx_total_histo[t][b] += cvsx->vsx_total_histo[t][b]; } } for (t = 0; t < ZIO_PRIORITY_NUM_QUEUEABLE; t++) { - for (b = 0; b < VDEV_HISTO_BUCKETS; b++) { + for (b = 0; b < ARRAY_SIZE(vsx->vsx_queue_histo[0]); b++) { vsx->vsx_queue_histo[t][b] += cvsx->vsx_queue_histo[t][b]; } vsx->vsx_active_queue[t] += cvsx->vsx_active_queue[t]; vsx->vsx_pend_queue[t] += cvsx->vsx_pend_queue[t]; + + for (b = 0; b < ARRAY_SIZE(vsx->vsx_ind_histo[0]); b++) + vsx->vsx_ind_histo[t][b] += cvsx->vsx_ind_histo[t][b]; + + for (b = 0; b < ARRAY_SIZE(vsx->vsx_agg_histo[0]); b++) + vsx->vsx_agg_histo[t][b] += cvsx->vsx_agg_histo[t][b]; } + } /* @@ -2974,13 +2987,21 @@ vdev_stat_update(zio_t *zio, uint64_t psize) vs->vs_ops[type]++; vs->vs_bytes[type] += psize; + if (flags & ZIO_FLAG_DELEGATED) { + vsx->vsx_agg_histo[zio->io_priority] + [RQ_HISTO(zio->io_size)]++; + } else { + vsx->vsx_ind_histo[zio->io_priority] + [RQ_HISTO(zio->io_size)]++; + } + if (zio->io_delta && zio->io_delay) { vsx->vsx_queue_histo[zio->io_priority] - [HISTO(zio->io_delta - zio->io_delay)]++; + [L_HISTO(zio->io_delta - zio->io_delay)]++; vsx->vsx_disk_histo[type] - [HISTO(zio->io_delay)]++; + [L_HISTO(zio->io_delay)]++; vsx->vsx_total_histo[type] - [HISTO(zio->io_delta)]++; + [L_HISTO(zio->io_delta)]++; } } diff --git a/module/zfs/vdev_disk.c b/module/zfs/vdev_disk.c index 4e362226a880..ccf1d839d99a 100644 --- a/module/zfs/vdev_disk.c +++ b/module/zfs/vdev_disk.c @@ -23,7 +23,7 @@ * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Rewritten for Linux by Brian Behlendorf . * LLNL-CODE-403049. - * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. */ #include @@ -41,10 +41,8 @@ static void *zfs_vdev_holder = VDEV_HOLDER; * Virtual device vector for disks. */ typedef struct dio_request { - struct completion dr_comp; /* Completion for sync IO */ zio_t *dr_zio; /* Parent ZIO */ atomic_t dr_ref; /* References */ - int dr_wait; /* Wait for IO */ int dr_error; /* Bio error */ int dr_bio_count; /* Count of bio's */ struct bio *dr_bio[0]; /* Attached bio's */ @@ -363,7 +361,6 @@ vdev_disk_dio_alloc(int bio_count) dr = kmem_zalloc(sizeof (dio_request_t) + sizeof (struct bio *) * bio_count, KM_SLEEP); if (dr) { - init_completion(&dr->dr_comp); atomic_set(&dr->dr_ref, 0); dr->dr_bio_count = bio_count; dr->dr_error = 0; @@ -414,7 +411,7 @@ vdev_disk_dio_put(dio_request_t *dr) ASSERT3S(zio->io_error, >=, 0); if (zio->io_error) vdev_disk_error(zio); - zio_interrupt(zio); + zio_delay_interrupt(zio); } } @@ -425,7 +422,6 @@ BIO_END_IO_PROTO(vdev_disk_physio_completion, bio, error) { dio_request_t *dr = bio->bi_private; int rc; - int wait; if (dr->dr_error == 0) { #ifdef HAVE_1ARG_BIO_END_IO_T @@ -438,13 +434,8 @@ BIO_END_IO_PROTO(vdev_disk_physio_completion, bio, error) #endif } - wait = dr->dr_wait; /* Drop reference aquired by __vdev_disk_physio */ rc = vdev_disk_dio_put(dr); - - /* Wake up synchronous waiter this is the last outstanding bio */ - if (wait && rc == 1) - complete(&dr->dr_comp); } static inline unsigned long @@ -493,30 +484,45 @@ bio_map(struct bio *bio, void *bio_ptr, unsigned int bio_size) return (bio_size); } +#ifndef bio_set_op_attrs +#define bio_set_op_attrs(bio, rw, flags) \ + do { (bio)->bi_rw |= (rw)|(flags); } while (0) +#endif + static inline void -vdev_submit_bio(int rw, struct bio *bio) +vdev_submit_bio_impl(struct bio *bio) +{ +#ifdef HAVE_1ARG_SUBMIT_BIO + submit_bio(bio); +#else + submit_bio(0, bio); +#endif +} + +static inline void +vdev_submit_bio(struct bio *bio) { #ifdef HAVE_CURRENT_BIO_TAIL struct bio **bio_tail = current->bio_tail; current->bio_tail = NULL; - submit_bio(rw, bio); + vdev_submit_bio_impl(bio); current->bio_tail = bio_tail; #else struct bio_list *bio_list = current->bio_list; current->bio_list = NULL; - submit_bio(rw, bio); + vdev_submit_bio_impl(bio); current->bio_list = bio_list; #endif } static int __vdev_disk_physio(struct block_device *bdev, zio_t *zio, caddr_t kbuf_ptr, - size_t kbuf_size, uint64_t kbuf_offset, int flags, int wait) + size_t kbuf_size, uint64_t kbuf_offset, int rw, int flags) { dio_request_t *dr; caddr_t bio_ptr; uint64_t bio_offset; - int rw, bio_size, bio_count = 16; + int bio_size, bio_count = 16; int i = 0, error = 0; ASSERT3U(kbuf_offset + kbuf_size, <=, bdev->bd_inode->i_size); @@ -529,9 +535,7 @@ __vdev_disk_physio(struct block_device *bdev, zio_t *zio, caddr_t kbuf_ptr, if (zio && !(zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD))) bio_set_flags_failfast(bdev, &flags); - rw = flags; dr->dr_zio = zio; - dr->dr_wait = wait; /* * When the IO size exceeds the maximum bio size for the request @@ -573,9 +577,9 @@ __vdev_disk_physio(struct block_device *bdev, zio_t *zio, caddr_t kbuf_ptr, dr->dr_bio[i]->bi_bdev = bdev; BIO_BI_SECTOR(dr->dr_bio[i]) = bio_offset >> 9; - dr->dr_bio[i]->bi_rw = rw; dr->dr_bio[i]->bi_end_io = vdev_disk_physio_completion; dr->dr_bio[i]->bi_private = dr; + bio_set_op_attrs(dr->dr_bio[i], rw, flags); /* Remaining size is returned to become the new size */ bio_size = bio_map(dr->dr_bio[i], bio_ptr, bio_size); @@ -591,34 +595,22 @@ __vdev_disk_physio(struct block_device *bdev, zio_t *zio, caddr_t kbuf_ptr, /* Submit all bio's associated with this dio */ for (i = 0; i < dr->dr_bio_count; i++) if (dr->dr_bio[i]) - vdev_submit_bio(rw, dr->dr_bio[i]); - - /* - * On synchronous blocking requests we wait for all bio the completion - * callbacks to run. We will be woken when the last callback runs - * for this dio. We are responsible for putting the last dio_request - * reference will in turn put back the last bio references. The - * only synchronous consumer is vdev_disk_read_rootlabel() all other - * IO originating from vdev_disk_io_start() is asynchronous. - */ - if (wait) { - wait_for_completion(&dr->dr_comp); - error = dr->dr_error; - ASSERT3S(atomic_read(&dr->dr_ref), ==, 1); - } + vdev_submit_bio(dr->dr_bio[i]); (void) vdev_disk_dio_put(dr); return (error); } +#ifndef __linux__ int vdev_disk_physio(struct block_device *bdev, caddr_t kbuf, - size_t size, uint64_t offset, int flags) + size_t size, uint64_t offset, int rw, int flags) { bio_set_flags_failfast(bdev, &flags); - return (__vdev_disk_physio(bdev, NULL, kbuf, size, offset, flags, 1)); + return (__vdev_disk_physio(bdev, NULL, kbuf, size, offset, rw, flags)); } +#endif BIO_END_IO_PROTO(vdev_disk_io_flush_completion, bio, rc) { @@ -656,7 +648,8 @@ vdev_disk_io_flush(struct block_device *bdev, zio_t *zio) bio->bi_end_io = vdev_disk_io_flush_completion; bio->bi_private = zio; bio->bi_bdev = bdev; - vdev_submit_bio(VDEV_WRITE_FLUSH_FUA, bio); + bio_set_op_attrs(bio, 0, VDEV_WRITE_FLUSH_FUA); + vdev_submit_bio(bio); invalidate_bdev(bdev); return (0); @@ -667,8 +660,7 @@ vdev_disk_io_start(zio_t *zio) { vdev_t *v = zio->io_vd; vdev_disk_t *vd = v->vdev_tsd; - zio_priority_t pri = zio->io_priority; - int flags, error; + int rw, flags, error; switch (zio->io_type) { case ZIO_TYPE_IOCTL: @@ -707,17 +699,25 @@ vdev_disk_io_start(zio_t *zio) zio_execute(zio); return; case ZIO_TYPE_WRITE: - if ((pri == ZIO_PRIORITY_SYNC_WRITE) && (v->vdev_nonrot)) - flags = WRITE_SYNC; - else - flags = WRITE; + rw = WRITE; +#if defined(HAVE_BLK_QUEUE_HAVE_BIO_RW_UNPLUG) + flags = (1 << BIO_RW_UNPLUG); +#elif defined(REQ_UNPLUG) + flags = REQ_UNPLUG; +#else + flags = 0; +#endif break; case ZIO_TYPE_READ: - if ((pri == ZIO_PRIORITY_SYNC_READ) && (v->vdev_nonrot)) - flags = READ_SYNC; - else - flags = READ; + rw = READ; +#if defined(HAVE_BLK_QUEUE_HAVE_BIO_RW_UNPLUG) + flags = (1 << BIO_RW_UNPLUG); +#elif defined(REQ_UNPLUG) + flags = REQ_UNPLUG; +#else + flags = 0; +#endif break; default: @@ -726,8 +726,9 @@ vdev_disk_io_start(zio_t *zio) return; } + zio->io_target_timestamp = zio_handle_io_delay(zio); error = __vdev_disk_physio(vd->vd_bdev, zio, zio->io_data, - zio->io_size, zio->io_offset, flags, 0); + zio->io_size, zio->io_offset, rw, flags); if (error) { zio->io_error = error; zio_interrupt(zio); @@ -797,6 +798,7 @@ vdev_ops_t vdev_disk_ops = { B_TRUE /* leaf vdev */ }; +#ifndef __linux__ /* * Given the root disk device devid or pathname, read the label from * the device, and construct a configuration nvlist. @@ -828,7 +830,8 @@ vdev_disk_read_rootlabel(char *devpath, char *devid, nvlist_t **config) /* read vdev label */ offset = vdev_label_offset(size, i, 0); if (vdev_disk_physio(bdev, (caddr_t)label, - VDEV_SKIP_SIZE + VDEV_PHYS_SIZE, offset, READ_SYNC) != 0) + VDEV_SKIP_SIZE + VDEV_PHYS_SIZE, offset, READ, + REQ_SYNC) != 0) continue; if (nvlist_unpack(label->vl_vdev_phys.vp_nvlist, @@ -859,6 +862,7 @@ vdev_disk_read_rootlabel(char *devpath, char *devid, nvlist_t **config) return (0); } +#endif /* __linux__ */ module_param(zfs_vdev_scheduler, charp, 0644); MODULE_PARM_DESC(zfs_vdev_scheduler, "I/O scheduler"); diff --git a/module/zfs/vdev_file.c b/module/zfs/vdev_file.c index a29ea7bf9515..bca4175a6ff4 100644 --- a/module/zfs/vdev_file.c +++ b/module/zfs/vdev_file.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2014 by Delphix. All rights reserved. + * Copyright (c) 2011, 2015 by Delphix. All rights reserved. */ #include @@ -159,7 +159,7 @@ vdev_file_io_strategy(void *arg) if (resid != 0 && zio->io_error == 0) zio->io_error = SET_ERROR(ENOSPC); - zio_interrupt(zio); + zio_delay_interrupt(zio); } static void @@ -217,6 +217,8 @@ vdev_file_io_start(zio_t *zio) return; } + zio->io_target_timestamp = zio_handle_io_delay(zio); + VERIFY3U(taskq_dispatch(system_taskq, vdev_file_io_strategy, zio, TQ_SLEEP), !=, 0); } diff --git a/module/zfs/vdev_label.c b/module/zfs/vdev_label.c index 1400aee7b757..1925c67edd1b 100644 --- a/module/zfs/vdev_label.c +++ b/module/zfs/vdev_label.c @@ -302,9 +302,51 @@ vdev_config_generate_stats(vdev_t *vd, nvlist_t *nv) vsx->vsx_queue_histo[ZIO_PRIORITY_SCRUB], ARRAY_SIZE(vsx->vsx_queue_histo[ZIO_PRIORITY_SCRUB])); + /* Request sizes */ + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_IND_R_HISTO, + vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_READ], + ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_READ])); + + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_IND_W_HISTO, + vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_WRITE], + ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_SYNC_WRITE])); + + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_IND_R_HISTO, + vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_READ], + ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_READ])); + + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_IND_W_HISTO, + vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_WRITE], + ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_ASYNC_WRITE])); + + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_IND_SCRUB_HISTO, + vsx->vsx_ind_histo[ZIO_PRIORITY_SCRUB], + ARRAY_SIZE(vsx->vsx_ind_histo[ZIO_PRIORITY_SCRUB])); + + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_AGG_R_HISTO, + vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_READ], + ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_READ])); + + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_SYNC_AGG_W_HISTO, + vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_WRITE], + ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_SYNC_WRITE])); + + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_AGG_R_HISTO, + vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_READ], + ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_READ])); + + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_ASYNC_AGG_W_HISTO, + vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_WRITE], + ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_ASYNC_WRITE])); + + fnvlist_add_uint64_array(nvx, ZPOOL_CONFIG_VDEV_AGG_SCRUB_HISTO, + vsx->vsx_agg_histo[ZIO_PRIORITY_SCRUB], + ARRAY_SIZE(vsx->vsx_agg_histo[ZIO_PRIORITY_SCRUB])); + /* Add extended stats nvlist to main nvlist */ fnvlist_add_nvlist(nv, ZPOOL_CONFIG_VDEV_STATS_EX, nvx); + fnvlist_free(nvx); kmem_free(vsx, sizeof (*vsx)); } diff --git a/module/zfs/vdev_queue.c b/module/zfs/vdev_queue.c index af8af67de70d..8522c2627fe6 100644 --- a/module/zfs/vdev_queue.c +++ b/module/zfs/vdev_queue.c @@ -249,20 +249,29 @@ static int vdev_queue_max_async_writes(spa_t *spa) { int writes; - uint64_t dirty = spa->spa_dsl_pool->dp_dirty_total; + uint64_t dirty = 0; + dsl_pool_t *dp = spa_get_dsl(spa); uint64_t min_bytes = zfs_dirty_data_max * zfs_vdev_async_write_active_min_dirty_percent / 100; uint64_t max_bytes = zfs_dirty_data_max * zfs_vdev_async_write_active_max_dirty_percent / 100; + /* + * Async writes may occur before the assignment of the spa's + * dsl_pool_t if a self-healing zio is issued prior to the + * completion of dmu_objset_open_impl(). + */ + if (dp == NULL) + return (zfs_vdev_async_write_max_active); + /* * Sync tasks correspond to interactive user actions. To reduce the * execution time of those actions we push data out as fast as possible. */ - if (spa_has_pending_synctask(spa)) { + if (spa_has_pending_synctask(spa)) return (zfs_vdev_async_write_max_active); - } + dirty = dp->dp_dirty_total; if (dirty < min_bytes) return (zfs_vdev_async_write_min_active); if (dirty > max_bytes) @@ -753,9 +762,6 @@ vdev_queue_io_done(zio_t *zio) vdev_queue_t *vq = &zio->io_vd->vdev_queue; zio_t *nio; - if (zio_injection_enabled) - delay(SEC_TO_TICK(zio_handle_io_delay(zio))); - mutex_enter(&vq->vq_lock); vdev_queue_pending_remove(vq, zio); diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index b9479092c842..b67de08966ff 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -22,6 +22,7 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2016 Gvozden Nešković. All rights reserved. */ #include @@ -31,6 +32,8 @@ #include #include #include +#include +#include /* * Virtual device vector for RAID-Z. @@ -99,34 +102,6 @@ * or in concert to recover missing data columns. */ -typedef struct raidz_col { - uint64_t rc_devidx; /* child device index for I/O */ - uint64_t rc_offset; /* device offset */ - uint64_t rc_size; /* I/O size */ - void *rc_data; /* I/O data */ - void *rc_gdata; /* used to store the "good" version */ - int rc_error; /* I/O error for this device */ - uint8_t rc_tried; /* Did we attempt this I/O column? */ - uint8_t rc_skipped; /* Did we skip this I/O column? */ -} raidz_col_t; - -typedef struct raidz_map { - uint64_t rm_cols; /* Regular column count */ - uint64_t rm_scols; /* Count including skipped columns */ - uint64_t rm_bigcols; /* Number of oversized columns */ - uint64_t rm_asize; /* Actual total I/O size */ - uint64_t rm_missingdata; /* Count of missing data devices */ - uint64_t rm_missingparity; /* Count of missing parity devices */ - uint64_t rm_firstdatacol; /* First data column/parity count */ - uint64_t rm_nskip; /* Skipped sectors for padding */ - uint64_t rm_skipstart; /* Column index of padding start */ - void *rm_datacopy; /* rm_asize-buffer of copied data */ - uintptr_t rm_reports; /* # of referencing checksum reports */ - uint8_t rm_freed; /* map no longer has referencing ZIO */ - uint8_t rm_ecksuminjected; /* checksum error was injected */ - raidz_col_t rm_col[1]; /* Flexible array of I/O columns */ -} raidz_map_t; - #define VDEV_RAIDZ_P 0 #define VDEV_RAIDZ_Q 1 #define VDEV_RAIDZ_R 2 @@ -154,104 +129,7 @@ typedef struct raidz_map { VDEV_RAIDZ_64MUL_2((x), mask); \ } -/* - * Force reconstruction to use the general purpose method. - */ -int vdev_raidz_default_to_general; - -/* Powers of 2 in the Galois field defined above. */ -static const uint8_t vdev_raidz_pow2[256] = { - 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, - 0x1d, 0x3a, 0x74, 0xe8, 0xcd, 0x87, 0x13, 0x26, - 0x4c, 0x98, 0x2d, 0x5a, 0xb4, 0x75, 0xea, 0xc9, - 0x8f, 0x03, 0x06, 0x0c, 0x18, 0x30, 0x60, 0xc0, - 0x9d, 0x27, 0x4e, 0x9c, 0x25, 0x4a, 0x94, 0x35, - 0x6a, 0xd4, 0xb5, 0x77, 0xee, 0xc1, 0x9f, 0x23, - 0x46, 0x8c, 0x05, 0x0a, 0x14, 0x28, 0x50, 0xa0, - 0x5d, 0xba, 0x69, 0xd2, 0xb9, 0x6f, 0xde, 0xa1, - 0x5f, 0xbe, 0x61, 0xc2, 0x99, 0x2f, 0x5e, 0xbc, - 0x65, 0xca, 0x89, 0x0f, 0x1e, 0x3c, 0x78, 0xf0, - 0xfd, 0xe7, 0xd3, 0xbb, 0x6b, 0xd6, 0xb1, 0x7f, - 0xfe, 0xe1, 0xdf, 0xa3, 0x5b, 0xb6, 0x71, 0xe2, - 0xd9, 0xaf, 0x43, 0x86, 0x11, 0x22, 0x44, 0x88, - 0x0d, 0x1a, 0x34, 0x68, 0xd0, 0xbd, 0x67, 0xce, - 0x81, 0x1f, 0x3e, 0x7c, 0xf8, 0xed, 0xc7, 0x93, - 0x3b, 0x76, 0xec, 0xc5, 0x97, 0x33, 0x66, 0xcc, - 0x85, 0x17, 0x2e, 0x5c, 0xb8, 0x6d, 0xda, 0xa9, - 0x4f, 0x9e, 0x21, 0x42, 0x84, 0x15, 0x2a, 0x54, - 0xa8, 0x4d, 0x9a, 0x29, 0x52, 0xa4, 0x55, 0xaa, - 0x49, 0x92, 0x39, 0x72, 0xe4, 0xd5, 0xb7, 0x73, - 0xe6, 0xd1, 0xbf, 0x63, 0xc6, 0x91, 0x3f, 0x7e, - 0xfc, 0xe5, 0xd7, 0xb3, 0x7b, 0xf6, 0xf1, 0xff, - 0xe3, 0xdb, 0xab, 0x4b, 0x96, 0x31, 0x62, 0xc4, - 0x95, 0x37, 0x6e, 0xdc, 0xa5, 0x57, 0xae, 0x41, - 0x82, 0x19, 0x32, 0x64, 0xc8, 0x8d, 0x07, 0x0e, - 0x1c, 0x38, 0x70, 0xe0, 0xdd, 0xa7, 0x53, 0xa6, - 0x51, 0xa2, 0x59, 0xb2, 0x79, 0xf2, 0xf9, 0xef, - 0xc3, 0x9b, 0x2b, 0x56, 0xac, 0x45, 0x8a, 0x09, - 0x12, 0x24, 0x48, 0x90, 0x3d, 0x7a, 0xf4, 0xf5, - 0xf7, 0xf3, 0xfb, 0xeb, 0xcb, 0x8b, 0x0b, 0x16, - 0x2c, 0x58, 0xb0, 0x7d, 0xfa, 0xe9, 0xcf, 0x83, - 0x1b, 0x36, 0x6c, 0xd8, 0xad, 0x47, 0x8e, 0x01 -}; -/* Logs of 2 in the Galois field defined above. */ -static const uint8_t vdev_raidz_log2[256] = { - 0x00, 0x00, 0x01, 0x19, 0x02, 0x32, 0x1a, 0xc6, - 0x03, 0xdf, 0x33, 0xee, 0x1b, 0x68, 0xc7, 0x4b, - 0x04, 0x64, 0xe0, 0x0e, 0x34, 0x8d, 0xef, 0x81, - 0x1c, 0xc1, 0x69, 0xf8, 0xc8, 0x08, 0x4c, 0x71, - 0x05, 0x8a, 0x65, 0x2f, 0xe1, 0x24, 0x0f, 0x21, - 0x35, 0x93, 0x8e, 0xda, 0xf0, 0x12, 0x82, 0x45, - 0x1d, 0xb5, 0xc2, 0x7d, 0x6a, 0x27, 0xf9, 0xb9, - 0xc9, 0x9a, 0x09, 0x78, 0x4d, 0xe4, 0x72, 0xa6, - 0x06, 0xbf, 0x8b, 0x62, 0x66, 0xdd, 0x30, 0xfd, - 0xe2, 0x98, 0x25, 0xb3, 0x10, 0x91, 0x22, 0x88, - 0x36, 0xd0, 0x94, 0xce, 0x8f, 0x96, 0xdb, 0xbd, - 0xf1, 0xd2, 0x13, 0x5c, 0x83, 0x38, 0x46, 0x40, - 0x1e, 0x42, 0xb6, 0xa3, 0xc3, 0x48, 0x7e, 0x6e, - 0x6b, 0x3a, 0x28, 0x54, 0xfa, 0x85, 0xba, 0x3d, - 0xca, 0x5e, 0x9b, 0x9f, 0x0a, 0x15, 0x79, 0x2b, - 0x4e, 0xd4, 0xe5, 0xac, 0x73, 0xf3, 0xa7, 0x57, - 0x07, 0x70, 0xc0, 0xf7, 0x8c, 0x80, 0x63, 0x0d, - 0x67, 0x4a, 0xde, 0xed, 0x31, 0xc5, 0xfe, 0x18, - 0xe3, 0xa5, 0x99, 0x77, 0x26, 0xb8, 0xb4, 0x7c, - 0x11, 0x44, 0x92, 0xd9, 0x23, 0x20, 0x89, 0x2e, - 0x37, 0x3f, 0xd1, 0x5b, 0x95, 0xbc, 0xcf, 0xcd, - 0x90, 0x87, 0x97, 0xb2, 0xdc, 0xfc, 0xbe, 0x61, - 0xf2, 0x56, 0xd3, 0xab, 0x14, 0x2a, 0x5d, 0x9e, - 0x84, 0x3c, 0x39, 0x53, 0x47, 0x6d, 0x41, 0xa2, - 0x1f, 0x2d, 0x43, 0xd8, 0xb7, 0x7b, 0xa4, 0x76, - 0xc4, 0x17, 0x49, 0xec, 0x7f, 0x0c, 0x6f, 0xf6, - 0x6c, 0xa1, 0x3b, 0x52, 0x29, 0x9d, 0x55, 0xaa, - 0xfb, 0x60, 0x86, 0xb1, 0xbb, 0xcc, 0x3e, 0x5a, - 0xcb, 0x59, 0x5f, 0xb0, 0x9c, 0xa9, 0xa0, 0x51, - 0x0b, 0xf5, 0x16, 0xeb, 0x7a, 0x75, 0x2c, 0xd7, - 0x4f, 0xae, 0xd5, 0xe9, 0xe6, 0xe7, 0xad, 0xe8, - 0x74, 0xd6, 0xf4, 0xea, 0xa8, 0x50, 0x58, 0xaf, -}; - -static void vdev_raidz_generate_parity(raidz_map_t *rm); - -/* - * Multiply a given number by 2 raised to the given power. - */ -static uint8_t -vdev_raidz_exp2(uint_t a, int exp) -{ - if (a == 0) - return (0); - - ASSERT(exp >= 0); - ASSERT(vdev_raidz_log2[a] > 0 || a == 1); - - exp += vdev_raidz_log2[a]; - if (exp > 255) - exp -= 255; - - return (vdev_raidz_pow2[exp]); -} - -static void +void vdev_raidz_map_free(raidz_map_t *rm) { int c; @@ -437,7 +315,7 @@ static const zio_vsd_ops_t vdev_raidz_vsd_ops = { * Avoid inlining the function to keep vdev_raidz_io_start(), which * is this functions only caller, as small as possible on the stack. */ -noinline static raidz_map_t * +noinline raidz_map_t * vdev_raidz_map_alloc(zio_t *zio, uint64_t unit_shift, uint64_t dcols, uint64_t nparity) { @@ -579,6 +457,10 @@ vdev_raidz_map_alloc(zio_t *zio, uint64_t unit_shift, uint64_t dcols, zio->io_vsd = rm; zio->io_vsd_ops = &vdev_raidz_vsd_ops; + + /* init RAIDZ parity ops */ + rm->rm_ops = vdev_raidz_math_get_ops(); + return (rm); } @@ -726,9 +608,13 @@ vdev_raidz_generate_parity_pqr(raidz_map_t *rm) * Generate RAID parity in the first virtual columns according to the number of * parity columns available. */ -static void +void vdev_raidz_generate_parity(raidz_map_t *rm) { + /* Generate using the new math implementation */ + if (vdev_raidz_math_generate(rm) != RAIDZ_ORIGINAL_IMPL) + return; + switch (rm->rm_firstdatacol) { case 1: vdev_raidz_generate_parity_p(rm); @@ -1392,12 +1278,12 @@ vdev_raidz_reconstruct_general(raidz_map_t *rm, int *tgts, int ntgts) return (code); } -static int -vdev_raidz_reconstruct(raidz_map_t *rm, int *t, int nt) +int +vdev_raidz_reconstruct(raidz_map_t *rm, const int *t, int nt) { int tgts[VDEV_RAIDZ_MAXPARITY], *dt; int ntgts; - int i, c; + int i, c, ret; int code; int nbadparity, nbaddata; int parity_valid[VDEV_RAIDZ_MAXPARITY]; @@ -1435,34 +1321,38 @@ vdev_raidz_reconstruct(raidz_map_t *rm, int *t, int nt) dt = &tgts[nbadparity]; + + /* Reconstruct using the new math implementation */ + ret = vdev_raidz_math_reconstruct(rm, parity_valid, dt, nbaddata); + if (ret != RAIDZ_ORIGINAL_IMPL) + return (ret); + /* * See if we can use any of our optimized reconstruction routines. */ - if (!vdev_raidz_default_to_general) { - switch (nbaddata) { - case 1: - if (parity_valid[VDEV_RAIDZ_P]) - return (vdev_raidz_reconstruct_p(rm, dt, 1)); + switch (nbaddata) { + case 1: + if (parity_valid[VDEV_RAIDZ_P]) + return (vdev_raidz_reconstruct_p(rm, dt, 1)); - ASSERT(rm->rm_firstdatacol > 1); + ASSERT(rm->rm_firstdatacol > 1); - if (parity_valid[VDEV_RAIDZ_Q]) - return (vdev_raidz_reconstruct_q(rm, dt, 1)); + if (parity_valid[VDEV_RAIDZ_Q]) + return (vdev_raidz_reconstruct_q(rm, dt, 1)); - ASSERT(rm->rm_firstdatacol > 2); - break; + ASSERT(rm->rm_firstdatacol > 2); + break; - case 2: - ASSERT(rm->rm_firstdatacol > 1); + case 2: + ASSERT(rm->rm_firstdatacol > 1); - if (parity_valid[VDEV_RAIDZ_P] && - parity_valid[VDEV_RAIDZ_Q]) - return (vdev_raidz_reconstruct_pq(rm, dt, 2)); + if (parity_valid[VDEV_RAIDZ_P] && + parity_valid[VDEV_RAIDZ_Q]) + return (vdev_raidz_reconstruct_pq(rm, dt, 2)); - ASSERT(rm->rm_firstdatacol > 2); + ASSERT(rm->rm_firstdatacol > 2); - break; - } + break; } code = vdev_raidz_reconstruct_general(rm, tgts, ntgts); @@ -1739,11 +1629,6 @@ raidz_parity_verify(zio_t *zio, raidz_map_t *rm) return (ret); } -/* - * Keep statistics on all the ways that we used parity to correct data. - */ -static uint64_t raidz_corrected[1 << VDEV_RAIDZ_MAXPARITY]; - static int vdev_raidz_worst_error(raidz_map_t *rm) { @@ -1845,7 +1730,6 @@ vdev_raidz_combrec(zio_t *zio, int total_errors, int data_errors) */ code = vdev_raidz_reconstruct(rm, tgts, n); if (raidz_checksum_verify(zio) == 0) { - atomic_inc_64(&raidz_corrected[code]); for (i = 0; i < n; i++) { c = tgts[i]; @@ -2058,8 +1942,6 @@ vdev_raidz_io_done(zio_t *zio) code = vdev_raidz_reconstruct(rm, tgts, n); if (raidz_checksum_verify(zio) == 0) { - atomic_inc_64(&raidz_corrected[code]); - /* * If we read more parity disks than were used * for reconstruction, confirm that the other diff --git a/module/zfs/vdev_raidz_math.c b/module/zfs/vdev_raidz_math.c new file mode 100644 index 000000000000..2e57480c960c --- /dev/null +++ b/module/zfs/vdev_raidz_math.c @@ -0,0 +1,644 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. + */ + +#include +#include +#include +#include +#include + +#include +#include + +extern boolean_t raidz_will_scalar_work(void); + +/* Opaque implementation with NULL methods to represent original methods */ +static const raidz_impl_ops_t vdev_raidz_original_impl = { + .name = "original", + .is_supported = raidz_will_scalar_work, +}; + +/* RAIDZ parity op that contain the fastest methods */ +static raidz_impl_ops_t vdev_raidz_fastest_impl = { + .name = "fastest" +}; + +/* All compiled in implementations */ +const raidz_impl_ops_t *raidz_all_maths[] = { + &vdev_raidz_original_impl, + &vdev_raidz_scalar_impl, +#if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */ + &vdev_raidz_sse2_impl, +#endif +#if defined(__x86_64) && defined(HAVE_SSSE3) /* only x86_64 for now */ + &vdev_raidz_ssse3_impl, +#endif +#if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */ + &vdev_raidz_avx2_impl +#endif +}; + +/* Indicate that benchmark has been completed */ +static boolean_t raidz_math_initialized = B_FALSE; + +/* Select raidz implementation */ +#define IMPL_FASTEST (UINT32_MAX) +#define IMPL_CYCLE (UINT32_MAX - 1) +#define IMPL_ORIGINAL (0) +#define IMPL_SCALAR (1) + +#define RAIDZ_IMPL_READ(i) (*(volatile uint32_t *) &(i)) + +static uint32_t zfs_vdev_raidz_impl = IMPL_SCALAR; +static uint32_t user_sel_impl = IMPL_FASTEST; + +/* Hold all supported implementations */ +static size_t raidz_supp_impl_cnt = 0; +static raidz_impl_ops_t *raidz_supp_impl[ARRAY_SIZE(raidz_all_maths)]; + +/* + * kstats values for supported implementations + * Values represent per disk throughput of 8 disk+parity raidz vdev [B/s] + */ +static raidz_impl_kstat_t raidz_impl_kstats[ARRAY_SIZE(raidz_all_maths) + 1]; + +/* kstat for benchmarked implementations */ +static kstat_t *raidz_math_kstat = NULL; + +/* + * Selects the raidz operation for raidz_map + * If rm_ops is set to NULL original raidz implementation will be used + */ +raidz_impl_ops_t * +vdev_raidz_math_get_ops() +{ + raidz_impl_ops_t *ops = NULL; + const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl); + + switch (impl) { + case IMPL_FASTEST: + ASSERT(raidz_math_initialized); + ops = &vdev_raidz_fastest_impl; + break; +#if !defined(_KERNEL) + case IMPL_CYCLE: + { + ASSERT(raidz_math_initialized); + ASSERT3U(raidz_supp_impl_cnt, >, 0); + /* Cycle through all supported implementations */ + static size_t cycle_impl_idx = 0; + size_t idx = (++cycle_impl_idx) % raidz_supp_impl_cnt; + ops = raidz_supp_impl[idx]; + } + break; +#endif + case IMPL_ORIGINAL: + ops = (raidz_impl_ops_t *) &vdev_raidz_original_impl; + break; + case IMPL_SCALAR: + ops = (raidz_impl_ops_t *) &vdev_raidz_scalar_impl; + break; + default: + ASSERT(raidz_math_initialized); + ASSERT3U(impl, <, raidz_supp_impl_cnt); + ASSERT3U(raidz_supp_impl_cnt, >, 0); + ops = raidz_supp_impl[impl]; + break; + } + + ASSERT3P(ops, !=, NULL); + + return (ops); +} + +/* + * Select parity generation method for raidz_map + */ +int +vdev_raidz_math_generate(raidz_map_t *rm) +{ + raidz_gen_f gen_parity = NULL; + + switch (raidz_parity(rm)) { + case 1: + gen_parity = rm->rm_ops->gen[RAIDZ_GEN_P]; + break; + case 2: + gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQ]; + break; + case 3: + gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQR]; + break; + default: + gen_parity = NULL; + cmn_err(CE_PANIC, "invalid RAID-Z configuration %d", + raidz_parity(rm)); + break; + } + + /* if method is NULL execute the original implementation */ + if (gen_parity == NULL) + return (RAIDZ_ORIGINAL_IMPL); + + gen_parity(rm); + + return (0); +} + +static raidz_rec_f +reconstruct_fun_p_sel(raidz_map_t *rm, const int *parity_valid, + const int nbaddata) +{ + if (nbaddata == 1 && parity_valid[CODE_P]) { + return (rm->rm_ops->rec[RAIDZ_REC_P]); + } + return ((raidz_rec_f) NULL); +} + +static raidz_rec_f +reconstruct_fun_pq_sel(raidz_map_t *rm, const int *parity_valid, + const int nbaddata) +{ + if (nbaddata == 1) { + if (parity_valid[CODE_P]) { + return (rm->rm_ops->rec[RAIDZ_REC_P]); + } else if (parity_valid[CODE_Q]) { + return (rm->rm_ops->rec[RAIDZ_REC_Q]); + } + } else if (nbaddata == 2 && + parity_valid[CODE_P] && parity_valid[CODE_Q]) { + return (rm->rm_ops->rec[RAIDZ_REC_PQ]); + } + return ((raidz_rec_f) NULL); +} + +static raidz_rec_f +reconstruct_fun_pqr_sel(raidz_map_t *rm, const int *parity_valid, + const int nbaddata) +{ + if (nbaddata == 1) { + if (parity_valid[CODE_P]) { + return (rm->rm_ops->rec[RAIDZ_REC_P]); + } else if (parity_valid[CODE_Q]) { + return (rm->rm_ops->rec[RAIDZ_REC_Q]); + } else if (parity_valid[CODE_R]) { + return (rm->rm_ops->rec[RAIDZ_REC_R]); + } + } else if (nbaddata == 2) { + if (parity_valid[CODE_P] && parity_valid[CODE_Q]) { + return (rm->rm_ops->rec[RAIDZ_REC_PQ]); + } else if (parity_valid[CODE_P] && parity_valid[CODE_R]) { + return (rm->rm_ops->rec[RAIDZ_REC_PR]); + } else if (parity_valid[CODE_Q] && parity_valid[CODE_R]) { + return (rm->rm_ops->rec[RAIDZ_REC_QR]); + } + } else if (nbaddata == 3 && + parity_valid[CODE_P] && parity_valid[CODE_Q] && + parity_valid[CODE_R]) { + return (rm->rm_ops->rec[RAIDZ_REC_PQR]); + } + return ((raidz_rec_f) NULL); +} + +/* + * Select data reconstruction method for raidz_map + * @parity_valid - Parity validity flag + * @dt - Failed data index array + * @nbaddata - Number of failed data columns + */ +int +vdev_raidz_math_reconstruct(raidz_map_t *rm, const int *parity_valid, + const int *dt, const int nbaddata) +{ + raidz_rec_f rec_data = NULL; + + switch (raidz_parity(rm)) { + case PARITY_P: + rec_data = reconstruct_fun_p_sel(rm, parity_valid, nbaddata); + break; + case PARITY_PQ: + rec_data = reconstruct_fun_pq_sel(rm, parity_valid, nbaddata); + break; + case PARITY_PQR: + rec_data = reconstruct_fun_pqr_sel(rm, parity_valid, nbaddata); + break; + default: + cmn_err(CE_PANIC, "invalid RAID-Z configuration %d", + raidz_parity(rm)); + break; + } + + if (rec_data == NULL) + return (RAIDZ_ORIGINAL_IMPL); + else + return (rec_data(rm, dt)); +} + +const char *raidz_gen_name[] = { + "gen_p", "gen_pq", "gen_pqr" +}; +const char *raidz_rec_name[] = { + "rec_p", "rec_q", "rec_r", + "rec_pq", "rec_pr", "rec_qr", "rec_pqr" +}; + +#define RAIDZ_KSTAT_LINE_LEN (17 + 10*12 + 1) + +static int +raidz_math_kstat_headers(char *buf, size_t size) +{ + int i; + ssize_t off; + + ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN); + + off = snprintf(buf, size, "%-17s", "implementation"); + + for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) + off += snprintf(buf + off, size - off, "%-12s", + raidz_gen_name[i]); + + for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) + off += snprintf(buf + off, size - off, "%-12s", + raidz_rec_name[i]); + + (void) snprintf(buf + off, size - off, "\n"); + + return (0); +} + +static int +raidz_math_kstat_data(char *buf, size_t size, void *data) +{ + raidz_impl_kstat_t * fstat = &raidz_impl_kstats[raidz_supp_impl_cnt]; + raidz_impl_kstat_t * cstat = (raidz_impl_kstat_t *) data; + ssize_t off = 0; + int i; + + ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN); + + if (cstat == fstat) { + off += snprintf(buf + off, size - off, "%-17s", "fastest"); + + for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) { + int id = fstat->gen[i]; + off += snprintf(buf + off, size - off, "%-12s", + raidz_supp_impl[id]->name); + } + for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) { + int id = fstat->rec[i]; + off += snprintf(buf + off, size - off, "%-12s", + raidz_supp_impl[id]->name); + } + } else { + ptrdiff_t id = cstat - raidz_impl_kstats; + + off += snprintf(buf + off, size - off, "%-17s", + raidz_supp_impl[id]->name); + + for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) + off += snprintf(buf + off, size - off, "%-12llu", + (u_longlong_t) cstat->gen[i]); + + for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) + off += snprintf(buf + off, size - off, "%-12llu", + (u_longlong_t) cstat->rec[i]); + } + + (void) snprintf(buf + off, size - off, "\n"); + + return (0); +} + +static void * +raidz_math_kstat_addr(kstat_t *ksp, loff_t n) +{ + if (n <= raidz_supp_impl_cnt) + ksp->ks_private = (void *) (raidz_impl_kstats + n); + else + ksp->ks_private = NULL; + + return (ksp->ks_private); +} + +#define BENCH_D_COLS (8ULL) +#define BENCH_COLS (BENCH_D_COLS + PARITY_PQR) +#define BENCH_ZIO_SIZE (1ULL << SPA_OLD_MAXBLOCKSHIFT) /* 128 kiB */ +#define BENCH_NS MSEC2NSEC(25) /* 25ms */ + +typedef void (*benchmark_fn)(raidz_map_t *rm, const int fn); + +static void +benchmark_gen_impl(raidz_map_t *rm, const int fn) +{ + (void) fn; + vdev_raidz_generate_parity(rm); +} + +static void +benchmark_rec_impl(raidz_map_t *rm, const int fn) +{ + static const int rec_tgt[7][3] = { + {1, 2, 3}, /* rec_p: bad QR & D[0] */ + {0, 2, 3}, /* rec_q: bad PR & D[0] */ + {0, 1, 3}, /* rec_r: bad PQ & D[0] */ + {2, 3, 4}, /* rec_pq: bad R & D[0][1] */ + {1, 3, 4}, /* rec_pr: bad Q & D[0][1] */ + {0, 3, 4}, /* rec_qr: bad P & D[0][1] */ + {3, 4, 5} /* rec_pqr: bad & D[0][1][2] */ + }; + + vdev_raidz_reconstruct(rm, rec_tgt[fn], 3); +} + +/* + * Benchmarking of all supported implementations (raidz_supp_impl_cnt) + * is performed by setting the rm_ops pointer and calling the top level + * generate/reconstruct methods of bench_rm. + */ +static void +benchmark_raidz_impl(raidz_map_t *bench_rm, const int fn, benchmark_fn bench_fn) +{ + uint64_t run_cnt, speed, best_speed = 0; + hrtime_t t_start, t_diff; + raidz_impl_ops_t *curr_impl; + raidz_impl_kstat_t * fstat = &raidz_impl_kstats[raidz_supp_impl_cnt]; + int impl, i; + + for (impl = 0; impl < raidz_supp_impl_cnt; impl++) { + /* set an implementation to benchmark */ + curr_impl = raidz_supp_impl[impl]; + bench_rm->rm_ops = curr_impl; + + run_cnt = 0; + t_start = gethrtime(); + + do { + for (i = 0; i < 25; i++, run_cnt++) + bench_fn(bench_rm, fn); + + t_diff = gethrtime() - t_start; + } while (t_diff < BENCH_NS); + + speed = run_cnt * BENCH_ZIO_SIZE * NANOSEC; + speed /= (t_diff * BENCH_COLS); + + if (bench_fn == benchmark_gen_impl) + raidz_impl_kstats[impl].gen[fn] = speed; + else + raidz_impl_kstats[impl].rec[fn] = speed; + + /* Update fastest implementation method */ + if (speed > best_speed) { + best_speed = speed; + + if (bench_fn == benchmark_gen_impl) { + fstat->gen[fn] = impl; + vdev_raidz_fastest_impl.gen[fn] = + curr_impl->gen[fn]; + } else { + fstat->rec[fn] = impl; + vdev_raidz_fastest_impl.rec[fn] = + curr_impl->rec[fn]; + } + } + } +} + +void +vdev_raidz_math_init(void) +{ + raidz_impl_ops_t *curr_impl; + zio_t *bench_zio = NULL; + raidz_map_t *bench_rm = NULL; + uint64_t bench_parity; + int i, c, fn; + + /* move supported impl into raidz_supp_impl */ + for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) { + curr_impl = (raidz_impl_ops_t *) raidz_all_maths[i]; + + /* initialize impl */ + if (curr_impl->init) + curr_impl->init(); + + if (curr_impl->is_supported()) + raidz_supp_impl[c++] = (raidz_impl_ops_t *) curr_impl; + } + membar_producer(); /* complete raidz_supp_impl[] init */ + raidz_supp_impl_cnt = c; /* number of supported impl */ + +#if !defined(_KERNEL) + /* Skip benchmarking and use last implementation as fastest */ + memcpy(&vdev_raidz_fastest_impl, raidz_supp_impl[raidz_supp_impl_cnt-1], + sizeof (vdev_raidz_fastest_impl)); + strcpy(vdev_raidz_fastest_impl.name, "fastest"); + + raidz_math_initialized = B_TRUE; + + /* Use 'cycle' math selection method for userspace */ + VERIFY0(vdev_raidz_impl_set("cycle")); + return; +#endif + + /* Fake an zio and run the benchmark on it */ + bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP); + bench_zio->io_offset = 0; + bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */ + bench_zio->io_data = zio_data_buf_alloc(BENCH_ZIO_SIZE); + VERIFY(bench_zio->io_data); + memset(bench_zio->io_data, 0xAA, BENCH_ZIO_SIZE); /* warm up */ + + /* Benchmark parity generation methods */ + for (fn = 0; fn < RAIDZ_GEN_NUM; fn++) { + bench_parity = fn + 1; + /* New raidz_map is needed for each generate_p/q/r */ + bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT, + BENCH_D_COLS + bench_parity, bench_parity); + + benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl); + + vdev_raidz_map_free(bench_rm); + } + + /* Benchmark data reconstruction methods */ + bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT, + BENCH_COLS, PARITY_PQR); + + for (fn = 0; fn < RAIDZ_REC_NUM; fn++) + benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl); + + vdev_raidz_map_free(bench_rm); + + /* cleanup the bench zio */ + zio_data_buf_free(bench_zio->io_data, BENCH_ZIO_SIZE); + kmem_free(bench_zio, sizeof (zio_t)); + + /* install kstats for all impl */ + raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc", + KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); + + if (raidz_math_kstat != NULL) { + raidz_math_kstat->ks_data = NULL; + raidz_math_kstat->ks_ndata = UINT32_MAX; + kstat_set_raw_ops(raidz_math_kstat, + raidz_math_kstat_headers, + raidz_math_kstat_data, + raidz_math_kstat_addr); + kstat_install(raidz_math_kstat); + } + + /* Finish initialization */ + atomic_swap_32(&zfs_vdev_raidz_impl, user_sel_impl); + raidz_math_initialized = B_TRUE; +} + +void +vdev_raidz_math_fini(void) +{ + raidz_impl_ops_t const *curr_impl; + int i; + + if (raidz_math_kstat != NULL) { + kstat_delete(raidz_math_kstat); + raidz_math_kstat = NULL; + } + + /* fini impl */ + for (i = 0; i < ARRAY_SIZE(raidz_all_maths); i++) { + curr_impl = raidz_all_maths[i]; + if (curr_impl->fini) + curr_impl->fini(); + } +} + +static const struct { + char *name; + uint32_t sel; +} math_impl_opts[] = { +#if !defined(_KERNEL) + { "cycle", IMPL_CYCLE }, +#endif + { "fastest", IMPL_FASTEST }, + { "original", IMPL_ORIGINAL }, + { "scalar", IMPL_SCALAR } +}; + +/* + * Function sets desired raidz implementation. + * + * If we are called before init(), user preference will be saved in + * user_sel_impl, and applied in later init() call. This occurs when module + * parameter is specified on module load. Otherwise, directly update + * zfs_vdev_raidz_impl. + * + * @val Name of raidz implementation to use + * @param Unused. + */ +static int +zfs_vdev_raidz_impl_set(const char *val, struct kernel_param *kp) +{ + int err = -EINVAL; + char req_name[RAIDZ_IMPL_NAME_MAX]; + uint32_t impl = RAIDZ_IMPL_READ(user_sel_impl); + size_t i; + + /* sanitize input */ + i = strnlen(val, RAIDZ_IMPL_NAME_MAX); + if (i == 0 || i == RAIDZ_IMPL_NAME_MAX) + return (err); + + strlcpy(req_name, val, RAIDZ_IMPL_NAME_MAX); + while (i > 0 && !!isspace(req_name[i-1])) + i--; + req_name[i] = '\0'; + + /* Check mandatory options */ + for (i = 0; i < ARRAY_SIZE(math_impl_opts); i++) { + if (strcmp(req_name, math_impl_opts[i].name) == 0) { + impl = math_impl_opts[i].sel; + err = 0; + break; + } + } + + /* check all supported impl if init() was already called */ + if (err != 0 && raidz_math_initialized) { + /* check all supported implementations */ + for (i = 0; i < raidz_supp_impl_cnt; i++) { + if (strcmp(req_name, raidz_supp_impl[i]->name) == 0) { + impl = i; + err = 0; + break; + } + } + } + + if (err == 0) { + if (raidz_math_initialized) + atomic_swap_32(&zfs_vdev_raidz_impl, impl); + else + atomic_swap_32(&user_sel_impl, impl); + } + + return (err); +} + +int +vdev_raidz_impl_set(const char *val) +{ + ASSERT(raidz_math_initialized); + + return (zfs_vdev_raidz_impl_set(val, NULL)); +} + +#if defined(_KERNEL) && defined(HAVE_SPL) +static int +zfs_vdev_raidz_impl_get(char *buffer, struct kernel_param *kp) +{ + int i, cnt = 0; + char *fmt; + const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl); + + ASSERT(raidz_math_initialized); + + /* list mandatory options */ + for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 2; i++) { + fmt = (impl == math_impl_opts[i].sel) ? "[%s] " : "%s "; + cnt += sprintf(buffer + cnt, fmt, math_impl_opts[i].name); + } + + /* list all supported implementations */ + for (i = 0; i < raidz_supp_impl_cnt; i++) { + fmt = (i == impl) ? "[%s] " : "%s "; + cnt += sprintf(buffer + cnt, fmt, raidz_supp_impl[i]->name); + } + + return (cnt); +} + +module_param_call(zfs_vdev_raidz_impl, zfs_vdev_raidz_impl_set, + zfs_vdev_raidz_impl_get, NULL, 0644); +MODULE_PARM_DESC(zfs_vdev_raidz_impl, "Select raidz implementation."); +#endif diff --git a/module/zfs/vdev_raidz_math_avx2.c b/module/zfs/vdev_raidz_math_avx2.c new file mode 100644 index 000000000000..9ca1688c1f49 --- /dev/null +++ b/module/zfs/vdev_raidz_math_avx2.c @@ -0,0 +1,396 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. + */ + +#include + +#if defined(__x86_64) && defined(HAVE_AVX2) + +#include +#include + +#define __asm __asm__ __volatile__ + +#define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N +#define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) + +#define VR0_(REG, ...) "ymm"#REG +#define VR1_(_1, REG, ...) "ymm"#REG +#define VR2_(_1, _2, REG, ...) "ymm"#REG +#define VR3_(_1, _2, _3, REG, ...) "ymm"#REG +#define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG +#define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG +#define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG +#define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG + +#define VR0(r...) VR0_(r) +#define VR1(r...) VR1_(r) +#define VR2(r...) VR2_(r, 1) +#define VR3(r...) VR3_(r, 1, 2) +#define VR4(r...) VR4_(r, 1, 2) +#define VR5(r...) VR5_(r, 1, 2, 3) +#define VR6(r...) VR6_(r, 1, 2, 3, 4) +#define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) + +#define R_01(REG1, REG2, ...) REG1, REG2 +#define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 +#define R_23(REG...) _R_23(REG, 1, 2, 3) + +#define ASM_BUG() ASSERT(0) + +extern const uint8_t gf_clmul_mod_lt[4*256][16]; + +#define ELEM_SIZE 32 + +typedef struct v { + uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); +} v_t; + +#define PREFETCHNTA(ptr, offset) \ +{ \ + __asm( \ + "prefetchnta " #offset "(%[MEM])\n" \ + : : [MEM] "r" (ptr)); \ +} + +#define PREFETCH(ptr, offset) \ +{ \ + __asm( \ + "prefetcht0 " #offset "(%[MEM])\n" \ + : : [MEM] "r" (ptr)); \ +} + +#define XOR_ACC(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ + "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ + "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ + "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ + "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define XOR(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ + "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ + "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ + "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ + "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define COPY(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "vmovdqa %" VR0(r) ", %" VR4(r) "\n" \ + "vmovdqa %" VR1(r) ", %" VR5(r) "\n" \ + "vmovdqa %" VR2(r) ", %" VR6(r) "\n" \ + "vmovdqa %" VR3(r) ", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "vmovdqa %" VR0(r) ", %" VR2(r) "\n" \ + "vmovdqa %" VR1(r) ", %" VR3(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define LOAD(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ + "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ + "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n" \ + "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ + "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define STORE(dst, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ + "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ + "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n" \ + "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + case 2: \ + __asm( \ + "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ + "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define FLUSH() \ +{ \ + __asm("vzeroupper"); \ +} + +#define MUL2_SETUP() \ +{ \ + __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \ + __asm("vpbroadcastq %xmm14, %ymm14"); \ + __asm("vpxor %ymm15, %ymm15 ,%ymm15"); \ +} + +#define _MUL2(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + __asm( \ + "vpcmpgtb %" VR0(r)", %ymm15, %ymm12\n" \ + "vpcmpgtb %" VR1(r)", %ymm15, %ymm13\n" \ + "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \ + "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \ + "vpand %ymm14, %ymm12, %ymm12\n" \ + "vpand %ymm14, %ymm13, %ymm13\n" \ + "vpxor %ymm12, %" VR0(r)", %" VR0(r) "\n" \ + "vpxor %ymm13, %" VR1(r)", %" VR1(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL2(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + _MUL2(R_01(r)); \ + _MUL2(R_23(r)); \ + break; \ + case 2: \ + _MUL2(r); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL4(r...) \ +{ \ + MUL2(r); \ + MUL2(r); \ +} + +#define _0f "ymm15" +#define _as "ymm14" +#define _bs "ymm13" +#define _ltmod "ymm12" +#define _ltmul "ymm11" +#define _ta "ymm10" +#define _tb "ymm15" + +static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F; + +#define _MULx2(c, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + __asm( \ + "vpbroadcastb (%[mask]), %%" _0f "\n" \ + /* upper bits */ \ + "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n" \ + "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n" \ + \ + "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \ + "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \ + "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \ + "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \ + "vpand %%" _0f ", %%" _as ", %%" _as "\n" \ + "vpand %%" _0f ", %%" _bs ", %%" _bs "\n" \ + \ + "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \ + "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \ + "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \ + "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \ + /* lower bits */ \ + "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n" \ + "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n" \ + \ + "vpxor %%" _ta ", %%" _as ", %%" _as "\n" \ + "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n" \ + \ + "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \ + "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \ + "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\ + "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\ + \ + "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \ + "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \ + "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \ + "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \ + : : [mask] "r" (&_mul_mask), \ + [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL(c, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + _MULx2(c, R_01(r)); \ + _MULx2(c, R_23(r)); \ + break; \ + case 2: \ + _MULx2(c, R_01(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define raidz_math_begin() kfpu_begin() +#define raidz_math_end() \ +{ \ + FLUSH(); \ + kfpu_end(); \ +} + +#define GEN_P_DEFINE() {} +#define GEN_P_STRIDE 4 +#define GEN_P_P 0, 1, 2, 3 + +#define GEN_PQ_DEFINE() {} +#define GEN_PQ_STRIDE 4 +#define GEN_PQ_D 0, 1, 2, 3 +#define GEN_PQ_P 4, 5, 6, 7 +#define GEN_PQ_Q 8, 9, 10, 11 + +#define GEN_PQR_DEFINE() {} +#define GEN_PQR_STRIDE 2 +#define GEN_PQR_D 0, 1 +#define GEN_PQR_P 2, 3 +#define GEN_PQR_Q 4, 5 +#define GEN_PQR_R 6, 7 + +#define REC_P_DEFINE() {} +#define REC_P_STRIDE 4 +#define REC_P_X 0, 1, 2, 3 + +#define REC_Q_DEFINE() {} +#define REC_Q_STRIDE 4 +#define REC_Q_X 0, 1, 2, 3 + +#define REC_R_DEFINE() {} +#define REC_R_STRIDE 4 +#define REC_R_X 0, 1, 2, 3 + +#define REC_PQ_DEFINE() {} +#define REC_PQ_STRIDE 2 +#define REC_PQ_X 0, 1 +#define REC_PQ_Y 2, 3 +#define REC_PQ_D 4, 5 + +#define REC_PR_DEFINE() {} +#define REC_PR_STRIDE 2 +#define REC_PR_X 0, 1 +#define REC_PR_Y 2, 3 +#define REC_PR_D 4, 5 + +#define REC_QR_DEFINE() {} +#define REC_QR_STRIDE 2 +#define REC_QR_X 0, 1 +#define REC_QR_Y 2, 3 +#define REC_QR_D 4, 5 + +#define REC_PQR_DEFINE() {} +#define REC_PQR_STRIDE 2 +#define REC_PQR_X 0, 1 +#define REC_PQR_Y 2, 3 +#define REC_PQR_Z 4, 5 +#define REC_PQR_D 6, 7 +#define REC_PQR_XS 6, 7 +#define REC_PQR_YS 8, 9 + + +#include +#include "vdev_raidz_math_impl.h" + +DEFINE_GEN_METHODS(avx2); +DEFINE_REC_METHODS(avx2); + +static boolean_t +raidz_will_avx2_work(void) +{ + return (zfs_avx_available() && zfs_avx2_available()); +} + +const raidz_impl_ops_t vdev_raidz_avx2_impl = { + .init = NULL, + .fini = NULL, + .gen = RAIDZ_GEN_METHODS(avx2), + .rec = RAIDZ_REC_METHODS(avx2), + .is_supported = &raidz_will_avx2_work, + .name = "avx2" +}; + +#endif /* defined(__x86_64) && defined(HAVE_AVX2) */ diff --git a/module/zfs/vdev_raidz_math_impl.h b/module/zfs/vdev_raidz_math_impl.h new file mode 100644 index 000000000000..67cad76c1d32 --- /dev/null +++ b/module/zfs/vdev_raidz_math_impl.h @@ -0,0 +1,1199 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. + */ + +#ifndef _VDEV_RAIDZ_MATH_IMPL_H +#define _VDEV_RAIDZ_MATH_IMPL_H + +#include + +#define raidz_inline inline __attribute__((always_inline)) +#ifndef noinline +#define noinline __attribute__((noinline)) +#endif + +/* Calculate data offset in raidz column, offset is in bytes */ +#define COL_OFF(col, off) ((v_t *)(((char *)(col)->rc_data) + (off))) + +/* + * PARITY CALCULATION + * An optimized function is called for a full length of data columns + * If RAIDZ map contains remainder columns (shorter columns) the same function + * is called for reminder of full columns. + * + * GEN_[P|PQ|PQR]_BLOCK() functions are designed to be efficiently in-lined by + * the compiler. This removes a lot of conditionals from the inside loop which + * makes the code faster, especially for vectorized code. + * They are also highly parametrized, allowing for each implementation to define + * most optimal stride, and register allocation. + */ + +static raidz_inline void +GEN_P_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int ncols) +{ + int c; + size_t ioff; + raidz_col_t * const pcol = raidz_col_p(rm, CODE_P); + raidz_col_t *col; + + GEN_P_DEFINE(); + + for (ioff = off; ioff < end; ioff += (GEN_P_STRIDE * sizeof (v_t))) { + LOAD(COL_OFF(&(rm->rm_col[1]), ioff), GEN_P_P); + + for (c = 2; c < ncols; c++) { + col = &rm->rm_col[c]; + XOR_ACC(COL_OFF(col, ioff), GEN_P_P); + } + + STORE(COL_OFF(pcol, ioff), GEN_P_P); + } +} + +/* + * Generate P parity (RAIDZ1) + * + * @rm RAIDZ map + */ +static raidz_inline void +raidz_generate_p_impl(raidz_map_t * const rm) +{ + const int ncols = raidz_ncols(rm); + const size_t psize = raidz_big_size(rm); + const size_t short_size = raidz_short_size(rm); + + raidz_math_begin(); + + /* short_size */ + GEN_P_BLOCK(rm, 0, short_size, ncols); + + /* fullcols */ + GEN_P_BLOCK(rm, short_size, psize, raidz_nbigcols(rm)); + + raidz_math_end(); +} + +static raidz_inline void +GEN_PQ_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int ncols, const int nbigcols) +{ + int c; + size_t ioff; + raidz_col_t * const pcol = raidz_col_p(rm, CODE_P); + raidz_col_t * const qcol = raidz_col_p(rm, CODE_Q); + raidz_col_t *col; + + GEN_PQ_DEFINE(); + + MUL2_SETUP(); + + for (ioff = off; ioff < end; ioff += (GEN_PQ_STRIDE * sizeof (v_t))) { + LOAD(COL_OFF(&rm->rm_col[2], ioff), GEN_PQ_P); + COPY(GEN_PQ_P, GEN_PQ_Q); + + for (c = 3; c < nbigcols; c++) { + col = &rm->rm_col[c]; + LOAD(COL_OFF(col, ioff), GEN_PQ_D); + MUL2(GEN_PQ_Q); + XOR(GEN_PQ_D, GEN_PQ_P); + XOR(GEN_PQ_D, GEN_PQ_Q); + } + + STORE(COL_OFF(pcol, ioff), GEN_PQ_P); + + for (; c < ncols; c++) + MUL2(GEN_PQ_Q); + + STORE(COL_OFF(qcol, ioff), GEN_PQ_Q); + } +} + +/* + * Generate PQ parity (RAIDZ2) + * + * @rm RAIDZ map + */ +static raidz_inline void +raidz_generate_pq_impl(raidz_map_t * const rm) +{ + const int ncols = raidz_ncols(rm); + const size_t psize = raidz_big_size(rm); + const size_t short_size = raidz_short_size(rm); + + raidz_math_begin(); + + /* short_size */ + GEN_PQ_BLOCK(rm, 0, short_size, ncols, ncols); + + /* fullcols */ + GEN_PQ_BLOCK(rm, short_size, psize, ncols, raidz_nbigcols(rm)); + + raidz_math_end(); +} + + +static raidz_inline void +GEN_PQR_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int ncols, const int nbigcols) +{ + int c; + size_t ioff; + raidz_col_t *col; + raidz_col_t * const pcol = raidz_col_p(rm, CODE_P); + raidz_col_t * const qcol = raidz_col_p(rm, CODE_Q); + raidz_col_t * const rcol = raidz_col_p(rm, CODE_R); + + GEN_PQR_DEFINE(); + + MUL2_SETUP(); + + for (ioff = off; ioff < end; ioff += (GEN_PQR_STRIDE * sizeof (v_t))) { + LOAD(COL_OFF(&rm->rm_col[3], ioff), GEN_PQR_P); + COPY(GEN_PQR_P, GEN_PQR_Q); + COPY(GEN_PQR_P, GEN_PQR_R); + + for (c = 4; c < nbigcols; c++) { + col = &rm->rm_col[c]; + LOAD(COL_OFF(col, ioff), GEN_PQR_D); + MUL2(GEN_PQR_Q); + MUL4(GEN_PQR_R); + XOR(GEN_PQR_D, GEN_PQR_P); + XOR(GEN_PQR_D, GEN_PQR_Q); + XOR(GEN_PQR_D, GEN_PQR_R); + } + + STORE(COL_OFF(pcol, ioff), GEN_PQR_P); + + for (; c < ncols; c++) { + MUL2(GEN_PQR_Q); + MUL4(GEN_PQR_R); + } + + STORE(COL_OFF(qcol, ioff), GEN_PQR_Q); + STORE(COL_OFF(rcol, ioff), GEN_PQR_R); + } +} + + +/* + * Generate PQR parity (RAIDZ3) + * + * @rm RAIDZ map + */ +static raidz_inline void +raidz_generate_pqr_impl(raidz_map_t * const rm) +{ + const int ncols = raidz_ncols(rm); + const size_t psize = raidz_big_size(rm); + const size_t short_size = raidz_short_size(rm); + + raidz_math_begin(); + + /* short_size */ + GEN_PQR_BLOCK(rm, 0, short_size, ncols, ncols); + + /* fullcols */ + GEN_PQR_BLOCK(rm, short_size, psize, ncols, raidz_nbigcols(rm)); + + raidz_math_end(); +} + +/* + * DATA RECONSTRUCTION + * + * Data reconstruction process consists of two phases: + * - Syndrome calculation + * - Data reconstruction + * + * Syndrome is calculated by generating parity using available data columns + * and zeros in places of erasure. Existing parity is added to corresponding + * syndrome value to obtain the [P|Q|R]syn values from equation: + * P = Psyn + Dx + Dy + Dz + * Q = Qsyn + 2^x * Dx + 2^y * Dy + 2^z * Dz + * R = Rsyn + 4^x * Dx + 4^y * Dy + 4^z * Dz + * + * For data reconstruction phase, the corresponding equations are solved + * for missing data (Dx, Dy, Dz). This generally involves multiplying known + * symbols by an coefficient and adding them together. The multiplication + * constant coefficients are calculated ahead of the operation in + * raidz_rec_[q|r|pq|pq|qr|pqr]_coeff() functions. + * + * IMPLEMENTATION NOTE: RAID-Z block can have complex geometry, with "big" + * and "short" columns. + * For this reason, reconstruction is performed in minimum of + * two steps. First, from offset 0 to short_size, then from short_size to + * short_size. Calculation functions REC_[*]_BLOCK() are implemented to work + * over both ranges. The split also enables removal of conditional expressions + * from loop bodies, improving throughput of SIMD implementations. + * For the best performance, all functions marked with raidz_inline attribute + * must be inlined by compiler. + * + * parity data + * columns columns + * <----------> <------------------> + * x y <----+ missing columns (x, y) + * | | + * +---+---+---+---+-v-+---+-v-+---+ ^ 0 + * | | | | | | | | | | + * | | | | | | | | | | + * | P | Q | R | D | D | D | D | D | | + * | | | | 0 | 1 | 2 | 3 | 4 | | + * | | | | | | | | | v + * | | | | | +---+---+---+ ^ short_size + * | | | | | | | + * +---+---+---+---+---+ v big_size + * <------------------> <----------> + * big columns short columns + * + */ + +/* + * Functions calculate multiplication constants for data reconstruction. + * Coefficients depend on RAIDZ geometry, indexes of failed child vdevs, and + * used parity columns for reconstruction. + * @rm RAIDZ map + * @tgtidx array of missing data indexes + * @coeff output array of coefficients. Array must be user + * provided and must hold minimum MUL_CNT values + */ +static noinline void +raidz_rec_q_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff) +{ + const unsigned ncols = raidz_ncols(rm); + const unsigned x = tgtidx[TARGET_X]; + + coeff[MUL_Q_X] = gf_exp2(255 - (ncols - x - 1)); +} + +static noinline void +raidz_rec_r_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff) +{ + const unsigned ncols = raidz_ncols(rm); + const unsigned x = tgtidx[TARGET_X]; + + coeff[MUL_R_X] = gf_exp4(255 - (ncols - x - 1)); +} + +static noinline void +raidz_rec_pq_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff) +{ + const unsigned ncols = raidz_ncols(rm); + const unsigned x = tgtidx[TARGET_X]; + const unsigned y = tgtidx[TARGET_Y]; + gf_t a, b, e; + + a = gf_exp2(x + 255 - y); + b = gf_exp2(255 - (ncols - x - 1)); + e = a ^ 0x01; + + coeff[MUL_PQ_X] = gf_div(a, e); + coeff[MUL_PQ_Y] = gf_div(b, e); +} + +static noinline void +raidz_rec_pr_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff) +{ + const unsigned ncols = raidz_ncols(rm); + const unsigned x = tgtidx[TARGET_X]; + const unsigned y = tgtidx[TARGET_Y]; + + gf_t a, b, e; + + a = gf_exp4(x + 255 - y); + b = gf_exp4(255 - (ncols - x - 1)); + e = a ^ 0x01; + + coeff[MUL_PR_X] = gf_div(a, e); + coeff[MUL_PR_Y] = gf_div(b, e); +} + +static noinline void +raidz_rec_qr_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff) +{ + const unsigned ncols = raidz_ncols(rm); + const unsigned x = tgtidx[TARGET_X]; + const unsigned y = tgtidx[TARGET_Y]; + + gf_t nx, ny, nxxy, nxyy, d; + + nx = gf_exp2(ncols - x - 1); + ny = gf_exp2(ncols - y - 1); + nxxy = gf_mul(gf_mul(nx, nx), ny); + nxyy = gf_mul(gf_mul(nx, ny), ny); + d = nxxy ^ nxyy; + + coeff[MUL_QR_XQ] = ny; + coeff[MUL_QR_X] = gf_div(ny, d); + coeff[MUL_QR_YQ] = nx; + coeff[MUL_QR_Y] = gf_div(nx, d); +} + +static noinline void +raidz_rec_pqr_coeff(const raidz_map_t *rm, const int *tgtidx, unsigned *coeff) +{ + const unsigned ncols = raidz_ncols(rm); + const unsigned x = tgtidx[TARGET_X]; + const unsigned y = tgtidx[TARGET_Y]; + const unsigned z = tgtidx[TARGET_Z]; + + gf_t nx, ny, nz, nxx, nyy, nzz, nyyz, nyzz, xd, yd; + + nx = gf_exp2(ncols - x - 1); + ny = gf_exp2(ncols - y - 1); + nz = gf_exp2(ncols - z - 1); + + nxx = gf_exp4(ncols - x - 1); + nyy = gf_exp4(ncols - y - 1); + nzz = gf_exp4(ncols - z - 1); + + nyyz = gf_mul(gf_mul(ny, nz), ny); + nyzz = gf_mul(nzz, ny); + + xd = gf_mul(nxx, ny) ^ gf_mul(nx, nyy) ^ nyyz ^ + gf_mul(nxx, nz) ^ gf_mul(nzz, nx) ^ nyzz; + + yd = gf_inv(ny ^ nz); + + coeff[MUL_PQR_XP] = gf_div(nyyz ^ nyzz, xd); + coeff[MUL_PQR_XQ] = gf_div(nyy ^ nzz, xd); + coeff[MUL_PQR_XR] = gf_div(ny ^ nz, xd); + coeff[MUL_PQR_YU] = nx; + coeff[MUL_PQR_YP] = gf_mul(nz, yd); + coeff[MUL_PQR_YQ] = yd; +} + + +/* + * Reconstruction using P parity + * @rm RAIDZ map + * @off starting offset + * @end ending offset + * @x missing data column + * @ncols number of column + */ +static raidz_inline void +REC_P_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int x, const int ncols) +{ + int c; + size_t ioff; + const size_t firstdc = raidz_parity(rm); + raidz_col_t * const pcol = raidz_col_p(rm, CODE_P); + raidz_col_t * const xcol = raidz_col_p(rm, x); + raidz_col_t *col; + + REC_P_DEFINE(); + + for (ioff = off; ioff < end; ioff += (REC_P_STRIDE * sizeof (v_t))) { + LOAD(COL_OFF(pcol, ioff), REC_P_X); + + for (c = firstdc; c < x; c++) { + col = &rm->rm_col[c]; + XOR_ACC(COL_OFF(col, ioff), REC_P_X); + } + + for (c++; c < ncols; c++) { + col = &rm->rm_col[c]; + XOR_ACC(COL_OFF(col, ioff), REC_P_X); + } + + STORE(COL_OFF(xcol, ioff), REC_P_X); + } +} + +/* + * Reconstruct single data column using P parity + * @rec_method REC_P_BLOCK() + * + * @rm RAIDZ map + * @tgtidx array of missing data indexes + */ +static raidz_inline int +raidz_reconstruct_p_impl(raidz_map_t *rm, const int *tgtidx) +{ + const int x = tgtidx[TARGET_X]; + const int ncols = raidz_ncols(rm); + const int nbigcols = raidz_nbigcols(rm); + const size_t xsize = raidz_col_size(rm, x); + const size_t short_size = raidz_short_size(rm); + + raidz_math_begin(); + + /* 0 - short_size */ + REC_P_BLOCK(rm, 0, short_size, x, ncols); + + /* short_size - xsize */ + REC_P_BLOCK(rm, short_size, xsize, x, nbigcols); + + raidz_math_end(); + + return (1 << CODE_P); +} + +/* + * Reconstruct using Q parity + */ + +#define REC_Q_SYN_UPDATE() MUL2(REC_Q_X) + +#define REC_Q_INNER_LOOP(c) \ +{ \ + col = &rm->rm_col[c]; \ + REC_Q_SYN_UPDATE(); \ + XOR_ACC(COL_OFF(col, ioff), REC_Q_X); \ +} + +/* + * Reconstruction using Q parity + * @rm RAIDZ map + * @off starting offset + * @end ending offset + * @x missing data column + * @coeff multiplication coefficients + * @ncols number of column + * @nbigcols number of big columns + */ +static raidz_inline void +REC_Q_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int x, const unsigned *coeff, const int ncols, const int nbigcols) +{ + int c; + size_t ioff = 0; + const size_t firstdc = raidz_parity(rm); + raidz_col_t * const qcol = raidz_col_p(rm, CODE_Q); + raidz_col_t * const xcol = raidz_col_p(rm, x); + raidz_col_t *col; + + REC_Q_DEFINE(); + + for (ioff = off; ioff < end; ioff += (REC_Q_STRIDE * sizeof (v_t))) { + MUL2_SETUP(); + + XOR(REC_Q_X, REC_Q_X); + + if (ncols == nbigcols) { + for (c = firstdc; c < x; c++) + REC_Q_INNER_LOOP(c); + + REC_Q_SYN_UPDATE(); + for (c++; c < nbigcols; c++) + REC_Q_INNER_LOOP(c); + } else { + for (c = firstdc; c < nbigcols; c++) { + REC_Q_SYN_UPDATE(); + if (x != c) { + col = &rm->rm_col[c]; + XOR_ACC(COL_OFF(col, ioff), REC_Q_X); + } + } + for (; c < ncols; c++) + REC_Q_SYN_UPDATE(); + } + + XOR_ACC(COL_OFF(qcol, ioff), REC_Q_X); + MUL(coeff[MUL_Q_X], REC_Q_X); + STORE(COL_OFF(xcol, ioff), REC_Q_X); + } +} + +/* + * Reconstruct single data column using Q parity + * @rec_method REC_Q_BLOCK() + * + * @rm RAIDZ map + * @tgtidx array of missing data indexes + */ +static raidz_inline int +raidz_reconstruct_q_impl(raidz_map_t *rm, const int *tgtidx) +{ + const int x = tgtidx[TARGET_X]; + const int ncols = raidz_ncols(rm); + const int nbigcols = raidz_nbigcols(rm); + const size_t xsize = raidz_col_size(rm, x); + const size_t short_size = raidz_short_size(rm); + unsigned coeff[MUL_CNT]; + + raidz_rec_q_coeff(rm, tgtidx, coeff); + + raidz_math_begin(); + + /* 0 - short_size */ + REC_Q_BLOCK(rm, 0, short_size, x, coeff, ncols, ncols); + + /* short_size - xsize */ + REC_Q_BLOCK(rm, short_size, xsize, x, coeff, ncols, nbigcols); + + raidz_math_end(); + + return (1 << CODE_Q); +} + +/* + * Reconstruct using R parity + */ + +#define REC_R_SYN_UPDATE() MUL4(REC_R_X) +#define REC_R_INNER_LOOP(c) \ +{ \ + col = &rm->rm_col[c]; \ + REC_R_SYN_UPDATE(); \ + XOR_ACC(COL_OFF(col, ioff), REC_R_X); \ +} + +/* + * Reconstruction using R parity + * @rm RAIDZ map + * @off starting offset + * @end ending offset + * @x missing data column + * @coeff multiplication coefficients + * @ncols number of column + * @nbigcols number of big columns + */ +static raidz_inline void +REC_R_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int x, const unsigned *coeff, const int ncols, const int nbigcols) +{ + int c; + size_t ioff = 0; + const size_t firstdc = raidz_parity(rm); + raidz_col_t * const rcol = raidz_col_p(rm, CODE_R); + raidz_col_t * const xcol = raidz_col_p(rm, x); + raidz_col_t *col; + + REC_R_DEFINE(); + + for (ioff = off; ioff < end; ioff += (REC_R_STRIDE * sizeof (v_t))) { + MUL2_SETUP(); + + XOR(REC_R_X, REC_R_X); + + if (ncols == nbigcols) { + for (c = firstdc; c < x; c++) + REC_R_INNER_LOOP(c); + + REC_R_SYN_UPDATE(); + for (c++; c < nbigcols; c++) + REC_R_INNER_LOOP(c); + } else { + for (c = firstdc; c < nbigcols; c++) { + REC_R_SYN_UPDATE(); + if (c != x) { + col = &rm->rm_col[c]; + XOR_ACC(COL_OFF(col, ioff), REC_R_X); + } + } + for (; c < ncols; c++) + REC_R_SYN_UPDATE(); + } + + XOR_ACC(COL_OFF(rcol, ioff), REC_R_X); + MUL(coeff[MUL_R_X], REC_R_X); + STORE(COL_OFF(xcol, ioff), REC_R_X); + } +} + +/* + * Reconstruct single data column using R parity + * @rec_method REC_R_BLOCK() + * + * @rm RAIDZ map + * @tgtidx array of missing data indexes + */ +static raidz_inline int +raidz_reconstruct_r_impl(raidz_map_t *rm, const int *tgtidx) +{ + const int x = tgtidx[TARGET_X]; + const int ncols = raidz_ncols(rm); + const int nbigcols = raidz_nbigcols(rm); + const size_t xsize = raidz_col_size(rm, x); + const size_t short_size = raidz_short_size(rm); + unsigned coeff[MUL_CNT]; + + raidz_rec_r_coeff(rm, tgtidx, coeff); + + raidz_math_begin(); + + /* 0 - short_size */ + REC_R_BLOCK(rm, 0, short_size, x, coeff, ncols, ncols); + + /* short_size - xsize */ + REC_R_BLOCK(rm, short_size, xsize, x, coeff, ncols, nbigcols); + + raidz_math_end(); + + return (1 << CODE_R); +} + +/* + * Reconstruct using PQ parity + */ + +#define REC_PQ_SYN_UPDATE() MUL2(REC_PQ_Y) +#define REC_PQ_INNER_LOOP(c) \ +{ \ + col = &rm->rm_col[c]; \ + LOAD(COL_OFF(col, ioff), REC_PQ_D); \ + REC_PQ_SYN_UPDATE(); \ + XOR(REC_PQ_D, REC_PQ_X); \ + XOR(REC_PQ_D, REC_PQ_Y); \ +} + +/* + * Reconstruction using PQ parity + * @rm RAIDZ map + * @off starting offset + * @end ending offset + * @x missing data column + * @y missing data column + * @coeff multiplication coefficients + * @ncols number of column + * @nbigcols number of big columns + * @calcy calculate second data column + */ +static raidz_inline void +REC_PQ_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int x, const int y, const unsigned *coeff, const int ncols, + const int nbigcols, const boolean_t calcy) +{ + int c; + size_t ioff = 0; + const size_t firstdc = raidz_parity(rm); + raidz_col_t * const pcol = raidz_col_p(rm, CODE_P); + raidz_col_t * const qcol = raidz_col_p(rm, CODE_Q); + raidz_col_t * const xcol = raidz_col_p(rm, x); + raidz_col_t * const ycol = raidz_col_p(rm, y); + raidz_col_t *col; + + REC_PQ_DEFINE(); + + for (ioff = off; ioff < end; ioff += (REC_PQ_STRIDE * sizeof (v_t))) { + LOAD(COL_OFF(pcol, ioff), REC_PQ_X); + XOR(REC_PQ_Y, REC_PQ_Y); + MUL2_SETUP(); + + if (ncols == nbigcols) { + for (c = firstdc; c < x; c++) + REC_PQ_INNER_LOOP(c); + + REC_PQ_SYN_UPDATE(); + for (c++; c < y; c++) + REC_PQ_INNER_LOOP(c); + + REC_PQ_SYN_UPDATE(); + for (c++; c < nbigcols; c++) + REC_PQ_INNER_LOOP(c); + } else { + for (c = firstdc; c < nbigcols; c++) { + REC_PQ_SYN_UPDATE(); + if (c != x && c != y) { + col = &rm->rm_col[c]; + LOAD(COL_OFF(col, ioff), REC_PQ_D); + XOR(REC_PQ_D, REC_PQ_X); + XOR(REC_PQ_D, REC_PQ_Y); + } + } + for (; c < ncols; c++) + REC_PQ_SYN_UPDATE(); + } + + XOR_ACC(COL_OFF(qcol, ioff), REC_PQ_Y); + + /* Save Pxy */ + COPY(REC_PQ_X, REC_PQ_D); + + /* Calc X */ + MUL(coeff[MUL_PQ_X], REC_PQ_X); + MUL(coeff[MUL_PQ_Y], REC_PQ_Y); + XOR(REC_PQ_Y, REC_PQ_X); + STORE(COL_OFF(xcol, ioff), REC_PQ_X); + + if (calcy) { + /* Calc Y */ + XOR(REC_PQ_D, REC_PQ_X); + STORE(COL_OFF(ycol, ioff), REC_PQ_X); + } + } +} + +/* + * Reconstruct two data columns using PQ parity + * @rec_method REC_PQ_BLOCK() + * + * @rm RAIDZ map + * @tgtidx array of missing data indexes + */ +static raidz_inline int +raidz_reconstruct_pq_impl(raidz_map_t *rm, const int *tgtidx) +{ + const int x = tgtidx[TARGET_X]; + const int y = tgtidx[TARGET_Y]; + const int ncols = raidz_ncols(rm); + const int nbigcols = raidz_nbigcols(rm); + const size_t xsize = raidz_col_size(rm, x); + const size_t ysize = raidz_col_size(rm, y); + const size_t short_size = raidz_short_size(rm); + unsigned coeff[MUL_CNT]; + + raidz_rec_pq_coeff(rm, tgtidx, coeff); + + raidz_math_begin(); + + /* 0 - short_size */ + REC_PQ_BLOCK(rm, 0, short_size, x, y, coeff, ncols, ncols, B_TRUE); + + /* short_size - xsize */ + REC_PQ_BLOCK(rm, short_size, xsize, x, y, coeff, ncols, nbigcols, + xsize == ysize); + + raidz_math_end(); + + return ((1 << CODE_P) | (1 << CODE_Q)); +} + +/* + * Reconstruct using PR parity + */ + +#define REC_PR_SYN_UPDATE() MUL4(REC_PR_Y) +#define REC_PR_INNER_LOOP(c) \ +{ \ + col = &rm->rm_col[c]; \ + LOAD(COL_OFF(col, ioff), REC_PR_D); \ + REC_PR_SYN_UPDATE(); \ + XOR(REC_PR_D, REC_PR_X); \ + XOR(REC_PR_D, REC_PR_Y); \ +} + +/* + * Reconstruction using PR parity + * @rm RAIDZ map + * @off starting offset + * @end ending offset + * @x missing data column + * @y missing data column + * @coeff multiplication coefficients + * @ncols number of column + * @nbigcols number of big columns + * @calcy calculate second data column + */ +static raidz_inline void +REC_PR_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int x, const int y, const unsigned *coeff, const int ncols, + const int nbigcols, const boolean_t calcy) +{ + int c; + size_t ioff; + const size_t firstdc = raidz_parity(rm); + raidz_col_t * const pcol = raidz_col_p(rm, CODE_P); + raidz_col_t * const rcol = raidz_col_p(rm, CODE_R); + raidz_col_t * const xcol = raidz_col_p(rm, x); + raidz_col_t * const ycol = raidz_col_p(rm, y); + raidz_col_t *col; + + REC_PR_DEFINE(); + + for (ioff = off; ioff < end; ioff += (REC_PR_STRIDE * sizeof (v_t))) { + LOAD(COL_OFF(pcol, ioff), REC_PR_X); + XOR(REC_PR_Y, REC_PR_Y); + MUL2_SETUP(); + + if (ncols == nbigcols) { + for (c = firstdc; c < x; c++) + REC_PR_INNER_LOOP(c); + + REC_PR_SYN_UPDATE(); + for (c++; c < y; c++) + REC_PR_INNER_LOOP(c); + + REC_PR_SYN_UPDATE(); + for (c++; c < nbigcols; c++) + REC_PR_INNER_LOOP(c); + } else { + for (c = firstdc; c < nbigcols; c++) { + REC_PR_SYN_UPDATE(); + if (c != x && c != y) { + col = &rm->rm_col[c]; + LOAD(COL_OFF(col, ioff), REC_PR_D); + XOR(REC_PR_D, REC_PR_X); + XOR(REC_PR_D, REC_PR_Y); + } + } + for (; c < ncols; c++) + REC_PR_SYN_UPDATE(); + } + + XOR_ACC(COL_OFF(rcol, ioff), REC_PR_Y); + + /* Save Pxy */ + COPY(REC_PR_X, REC_PR_D); + + /* Calc X */ + MUL(coeff[MUL_PR_X], REC_PR_X); + MUL(coeff[MUL_PR_Y], REC_PR_Y); + XOR(REC_PR_Y, REC_PR_X); + STORE(COL_OFF(xcol, ioff), REC_PR_X); + + if (calcy) { + /* Calc Y */ + XOR(REC_PR_D, REC_PR_X); + STORE(COL_OFF(ycol, ioff), REC_PR_X); + } + } +} + + +/* + * Reconstruct two data columns using PR parity + * @rec_method REC_PR_BLOCK() + * + * @rm RAIDZ map + * @tgtidx array of missing data indexes + */ +static raidz_inline int +raidz_reconstruct_pr_impl(raidz_map_t *rm, const int *tgtidx) +{ + const int x = tgtidx[TARGET_X]; + const int y = tgtidx[TARGET_Y]; + const int ncols = raidz_ncols(rm); + const int nbigcols = raidz_nbigcols(rm); + const size_t xsize = raidz_col_size(rm, x); + const size_t ysize = raidz_col_size(rm, y); + const size_t short_size = raidz_short_size(rm); + unsigned coeff[MUL_CNT]; + + raidz_rec_pr_coeff(rm, tgtidx, coeff); + + raidz_math_begin(); + + /* 0 - short_size */ + REC_PR_BLOCK(rm, 0, short_size, x, y, coeff, ncols, ncols, B_TRUE); + + /* short_size - xsize */ + REC_PR_BLOCK(rm, short_size, xsize, x, y, coeff, ncols, nbigcols, + xsize == ysize); + + raidz_math_end(); + + return ((1 << CODE_P) | (1 << CODE_R)); +} + + +/* + * Reconstruct using QR parity + */ + +#define REC_QR_SYN_UPDATE() \ +{ \ + MUL2(REC_QR_X); \ + MUL4(REC_QR_Y); \ +} + +#define REC_QR_INNER_LOOP(c) \ +{ \ + col = &rm->rm_col[c]; \ + LOAD(COL_OFF(col, ioff), REC_QR_D); \ + REC_QR_SYN_UPDATE(); \ + XOR(REC_QR_D, REC_QR_X); \ + XOR(REC_QR_D, REC_QR_Y); \ +} + +/* + * Reconstruction using QR parity + * @rm RAIDZ map + * @off starting offset + * @end ending offset + * @x missing data column + * @y missing data column + * @coeff multiplication coefficients + * @ncols number of column + * @nbigcols number of big columns + * @calcy calculate second data column + */ +static raidz_inline void +REC_QR_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int x, const int y, const unsigned *coeff, const int ncols, + const int nbigcols, const boolean_t calcy) +{ + int c; + size_t ioff; + const size_t firstdc = raidz_parity(rm); + raidz_col_t * const qcol = raidz_col_p(rm, CODE_Q); + raidz_col_t * const rcol = raidz_col_p(rm, CODE_R); + raidz_col_t * const xcol = raidz_col_p(rm, x); + raidz_col_t * const ycol = raidz_col_p(rm, y); + raidz_col_t *col; + + REC_QR_DEFINE(); + + for (ioff = off; ioff < end; ioff += (REC_QR_STRIDE * sizeof (v_t))) { + MUL2_SETUP(); + XOR(REC_QR_X, REC_QR_X); + XOR(REC_QR_Y, REC_QR_Y); + + if (ncols == nbigcols) { + for (c = firstdc; c < x; c++) + REC_QR_INNER_LOOP(c); + + REC_QR_SYN_UPDATE(); + for (c++; c < y; c++) + REC_QR_INNER_LOOP(c); + + REC_QR_SYN_UPDATE(); + for (c++; c < nbigcols; c++) + REC_QR_INNER_LOOP(c); + } else { + for (c = firstdc; c < nbigcols; c++) { + REC_QR_SYN_UPDATE(); + if (c != x && c != y) { + col = &rm->rm_col[c]; + LOAD(COL_OFF(col, ioff), REC_QR_D); + XOR(REC_QR_D, REC_QR_X); + XOR(REC_QR_D, REC_QR_Y); + } + } + for (; c < ncols; c++) + REC_QR_SYN_UPDATE(); + } + + XOR_ACC(COL_OFF(qcol, ioff), REC_QR_X); + XOR_ACC(COL_OFF(rcol, ioff), REC_QR_Y); + + /* Save Qxy */ + COPY(REC_QR_X, REC_QR_D); + + /* Calc X */ + MUL(coeff[MUL_QR_XQ], REC_QR_X); /* X = Q * xqm */ + XOR(REC_QR_Y, REC_QR_X); /* X = R ^ X */ + MUL(coeff[MUL_QR_X], REC_QR_X); /* X = X * xm */ + STORE(COL_OFF(xcol, ioff), REC_QR_X); + + if (calcy) { + /* Calc Y */ + MUL(coeff[MUL_QR_YQ], REC_QR_D); /* X = Q * xqm */ + XOR(REC_QR_Y, REC_QR_D); /* X = R ^ X */ + MUL(coeff[MUL_QR_Y], REC_QR_D); /* X = X * xm */ + STORE(COL_OFF(ycol, ioff), REC_QR_D); + } + } +} + +/* + * Reconstruct two data columns using QR parity + * @rec_method REC_QR_BLOCK() + * + * @rm RAIDZ map + * @tgtidx array of missing data indexes + */ +static raidz_inline int +raidz_reconstruct_qr_impl(raidz_map_t *rm, const int *tgtidx) +{ + const int x = tgtidx[TARGET_X]; + const int y = tgtidx[TARGET_Y]; + const int ncols = raidz_ncols(rm); + const int nbigcols = raidz_nbigcols(rm); + const size_t xsize = raidz_col_size(rm, x); + const size_t ysize = raidz_col_size(rm, y); + const size_t short_size = raidz_short_size(rm); + unsigned coeff[MUL_CNT]; + + raidz_rec_qr_coeff(rm, tgtidx, coeff); + + raidz_math_begin(); + + /* 0 - short_size */ + REC_QR_BLOCK(rm, 0, short_size, x, y, coeff, ncols, ncols, B_TRUE); + + /* short_size - xsize */ + REC_QR_BLOCK(rm, short_size, xsize, x, y, coeff, ncols, nbigcols, + xsize == ysize); + + raidz_math_end(); + + return ((1 << CODE_Q) | (1 << CODE_R)); +} + +/* + * Reconstruct using PQR parity + */ + +#define REC_PQR_SYN_UPDATE() \ +{ \ + MUL2(REC_PQR_Y); \ + MUL4(REC_PQR_Z); \ +} + +#define REC_PQR_INNER_LOOP(c) \ +{ \ + col = &rm->rm_col[(c)]; \ + LOAD(COL_OFF(col, ioff), REC_PQR_D); \ + REC_PQR_SYN_UPDATE(); \ + XOR(REC_PQR_D, REC_PQR_X); \ + XOR(REC_PQR_D, REC_PQR_Y); \ + XOR(REC_PQR_D, REC_PQR_Z); \ +} + +/* + * Reconstruction using PQR parity + * @rm RAIDZ map + * @off starting offset + * @end ending offset + * @x missing data column + * @y missing data column + * @z missing data column + * @coeff multiplication coefficients + * @ncols number of column + * @nbigcols number of big columns + * @calcy calculate second data column + * @calcz calculate third data column + */ +static raidz_inline void +REC_PQR_BLOCK(raidz_map_t * const rm, const size_t off, const size_t end, + const int x, const int y, const int z, const unsigned *coeff, + const int ncols, const int nbigcols, const boolean_t calcy, + const boolean_t calcz) +{ + int c; + size_t ioff; + const size_t firstdc = raidz_parity(rm); + raidz_col_t * const pcol = raidz_col_p(rm, CODE_P); + raidz_col_t * const qcol = raidz_col_p(rm, CODE_Q); + raidz_col_t * const rcol = raidz_col_p(rm, CODE_R); + raidz_col_t * const xcol = raidz_col_p(rm, x); + raidz_col_t * const ycol = raidz_col_p(rm, y); + raidz_col_t * const zcol = raidz_col_p(rm, z); + raidz_col_t *col; + + REC_PQR_DEFINE(); + + for (ioff = off; ioff < end; ioff += (REC_PQR_STRIDE * sizeof (v_t))) { + MUL2_SETUP(); + LOAD(COL_OFF(pcol, ioff), REC_PQR_X); + XOR(REC_PQR_Y, REC_PQR_Y); + XOR(REC_PQR_Z, REC_PQR_Z); + + if (ncols == nbigcols) { + for (c = firstdc; c < x; c++) + REC_PQR_INNER_LOOP(c); + + REC_PQR_SYN_UPDATE(); + for (c++; c < y; c++) + REC_PQR_INNER_LOOP(c); + + REC_PQR_SYN_UPDATE(); + for (c++; c < z; c++) + REC_PQR_INNER_LOOP(c); + + REC_PQR_SYN_UPDATE(); + for (c++; c < nbigcols; c++) + REC_PQR_INNER_LOOP(c); + } else { + for (c = firstdc; c < nbigcols; c++) { + REC_PQR_SYN_UPDATE(); + if (c != x && c != y && c != z) { + col = &rm->rm_col[c]; + LOAD(COL_OFF(col, ioff), REC_PQR_D); + XOR(REC_PQR_D, REC_PQR_X); + XOR(REC_PQR_D, REC_PQR_Y); + XOR(REC_PQR_D, REC_PQR_Z); + } + } + for (; c < ncols; c++) + REC_PQR_SYN_UPDATE(); + } + + XOR_ACC(COL_OFF(qcol, ioff), REC_PQR_Y); + XOR_ACC(COL_OFF(rcol, ioff), REC_PQR_Z); + + /* Save Pxyz and Qxyz */ + COPY(REC_PQR_X, REC_PQR_XS); + COPY(REC_PQR_Y, REC_PQR_YS); + + /* Calc X */ + MUL(coeff[MUL_PQR_XP], REC_PQR_X); /* Xp = Pxyz * xp */ + MUL(coeff[MUL_PQR_XQ], REC_PQR_Y); /* Xq = Qxyz * xq */ + XOR(REC_PQR_Y, REC_PQR_X); + MUL(coeff[MUL_PQR_XR], REC_PQR_Z); /* Xr = Rxyz * xr */ + XOR(REC_PQR_Z, REC_PQR_X); /* X = Xp + Xq + Xr */ + STORE(COL_OFF(xcol, ioff), REC_PQR_X); + + if (calcy) { + /* Calc Y */ + XOR(REC_PQR_X, REC_PQR_XS); /* Pyz = Pxyz + X */ + MUL(coeff[MUL_PQR_YU], REC_PQR_X); /* Xq = X * upd_q */ + XOR(REC_PQR_X, REC_PQR_YS); /* Qyz = Qxyz + Xq */ + COPY(REC_PQR_XS, REC_PQR_X); /* restore Pyz */ + MUL(coeff[MUL_PQR_YP], REC_PQR_X); /* Yp = Pyz * yp */ + MUL(coeff[MUL_PQR_YQ], REC_PQR_YS); /* Yq = Qyz * yq */ + XOR(REC_PQR_X, REC_PQR_YS); /* Y = Yp + Yq */ + STORE(COL_OFF(ycol, ioff), REC_PQR_YS); + } + + if (calcz) { + /* Calc Z */ + XOR(REC_PQR_XS, REC_PQR_YS); /* Z = Pz = Pyz + Y */ + STORE(COL_OFF(zcol, ioff), REC_PQR_YS); + } + } +} + +/* + * Reconstruct three data columns using PQR parity + * @rec_method REC_PQR_BLOCK() + * + * @rm RAIDZ map + * @tgtidx array of missing data indexes + */ +static raidz_inline int +raidz_reconstruct_pqr_impl(raidz_map_t *rm, const int *tgtidx) +{ + const int x = tgtidx[TARGET_X]; + const int y = tgtidx[TARGET_Y]; + const int z = tgtidx[TARGET_Z]; + const int ncols = raidz_ncols(rm); + const int nbigcols = raidz_nbigcols(rm); + const size_t xsize = raidz_col_size(rm, x); + const size_t ysize = raidz_col_size(rm, y); + const size_t zsize = raidz_col_size(rm, z); + const size_t short_size = raidz_short_size(rm); + unsigned coeff[MUL_CNT]; + + raidz_rec_pqr_coeff(rm, tgtidx, coeff); + + raidz_math_begin(); + + /* 0 - short_size */ + REC_PQR_BLOCK(rm, 0, short_size, x, y, z, coeff, ncols, ncols, + B_TRUE, B_TRUE); + + /* short_size - xsize */ + REC_PQR_BLOCK(rm, short_size, xsize, x, y, z, coeff, ncols, nbigcols, + xsize == ysize, xsize == zsize); + + raidz_math_end(); + + return ((1 << CODE_P) | (1 << CODE_Q) | (1 << CODE_R)); +} + +#endif /* _VDEV_RAIDZ_MATH_IMPL_H */ diff --git a/module/zfs/vdev_raidz_math_scalar.c b/module/zfs/vdev_raidz_math_scalar.c new file mode 100644 index 000000000000..540cb9314fd4 --- /dev/null +++ b/module/zfs/vdev_raidz_math_scalar.c @@ -0,0 +1,310 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. + */ + +#include + +/* + * Provide native CPU scalar routines. + * Support 32bit and 64bit CPUs. + */ +#if ((~(0x0ULL)) >> 24) == 0xffULL +#define ELEM_SIZE 4 +typedef uint32_t iv_t; +#elif ((~(0x0ULL)) >> 56) == 0xffULL +#define ELEM_SIZE 8 +typedef uint64_t iv_t; +#endif + +/* + * Vector type used in scalar implementation + * + * The union is expected to be of native CPU register size. Since addition + * uses XOR operation, it can be performed an all byte elements at once. + * Multiplication requires per byte access. + */ +typedef union { + iv_t e; + uint8_t b[ELEM_SIZE]; +} v_t; + +/* + * Precomputed lookup tables for multiplication by a constant + * + * Reconstruction path requires multiplication by a constant factors. Instead of + * performing two step lookup (log & exp tables), a direct lookup can be used + * instead. Multiplication of element 'a' by a constant 'c' is obtained as: + * + * r = vdev_raidz_mul_lt[c_log][a]; + * + * where c_log = vdev_raidz_log2[c]. Log of coefficient factors is used because + * they are faster to obtain while solving the syndrome equations. + * + * PERFORMANCE NOTE: + * Even though the complete lookup table uses 64kiB, only relatively small + * portion of it is used at the same time. Following shows number of accessed + * bytes for different cases: + * - 1 failed disk: 256B (1 mul. coefficient) + * - 2 failed disks: 512B (2 mul. coefficients) + * - 3 failed disks: 1536B (6 mul. coefficients) + * + * Size of actually accessed lookup table regions is only larger for + * reconstruction of 3 failed disks, when compared to traditional log/exp + * method. But since the result is obtained in one lookup step performance is + * doubled. + */ +static uint8_t vdev_raidz_mul_lt[256][256] __attribute__((aligned(256))); + +static void +raidz_init_scalar(void) +{ + int c, i; + for (c = 0; c < 256; c++) + for (i = 0; i < 256; i++) + vdev_raidz_mul_lt[c][i] = gf_mul(c, i); + +} + +#define PREFETCHNTA(ptr, offset) {} +#define PREFETCH(ptr, offset) {} + +#define XOR_ACC(src, acc) acc.e ^= ((v_t *)src)[0].e +#define XOR(src, acc) acc.e ^= src.e +#define COPY(src, dst) dst = src +#define LOAD(src, val) val = ((v_t *)src)[0] +#define STORE(dst, val) ((v_t *)dst)[0] = val + +/* + * Constants used for optimized multiplication by 2. + */ +static const struct { + iv_t mod; + iv_t mask; + iv_t msb; +} scalar_mul2_consts = { +#if ELEM_SIZE == 8 + .mod = 0x1d1d1d1d1d1d1d1dULL, + .mask = 0xfefefefefefefefeULL, + .msb = 0x8080808080808080ULL, +#else + .mod = 0x1d1d1d1dULL, + .mask = 0xfefefefeULL, + .msb = 0x80808080ULL, +#endif +}; + +#define MUL2_SETUP() {} + +#define MUL2(a) \ +{ \ + iv_t _mask; \ + \ + _mask = (a).e & scalar_mul2_consts.msb; \ + _mask = (_mask << 1) - (_mask >> 7); \ + (a).e = ((a).e << 1) & scalar_mul2_consts.mask; \ + (a).e = (a).e ^ (_mask & scalar_mul2_consts.mod); \ +} + +#define MUL4(a) \ +{ \ + MUL2(a); \ + MUL2(a); \ +} + +#define MUL(c, a) \ +{ \ + const uint8_t *mul_lt = vdev_raidz_mul_lt[c]; \ + switch (ELEM_SIZE) { \ + case 8: \ + a.b[7] = mul_lt[a.b[7]]; \ + a.b[6] = mul_lt[a.b[6]]; \ + a.b[5] = mul_lt[a.b[5]]; \ + a.b[4] = mul_lt[a.b[4]]; \ + case 4: \ + a.b[3] = mul_lt[a.b[3]]; \ + a.b[2] = mul_lt[a.b[2]]; \ + a.b[1] = mul_lt[a.b[1]]; \ + a.b[0] = mul_lt[a.b[0]]; \ + break; \ + } \ +} + +#define raidz_math_begin() {} +#define raidz_math_end() {} + +#define GEN_P_DEFINE() v_t p0 +#define GEN_P_STRIDE 1 +#define GEN_P_P p0 + +#define GEN_PQ_DEFINE() v_t d0, p0, q0 +#define GEN_PQ_STRIDE 1 +#define GEN_PQ_D d0 +#define GEN_PQ_P p0 +#define GEN_PQ_Q q0 + +#define GEN_PQR_DEFINE() v_t d0, p0, q0, r0 +#define GEN_PQR_STRIDE 1 +#define GEN_PQR_D d0 +#define GEN_PQR_P p0 +#define GEN_PQR_Q q0 +#define GEN_PQR_R r0 + +#define REC_P_DEFINE() v_t x0 +#define REC_P_STRIDE 1 +#define REC_P_X x0 + +#define REC_Q_DEFINE() v_t x0 +#define REC_Q_STRIDE 1 +#define REC_Q_X x0 + +#define REC_R_DEFINE() v_t x0 +#define REC_R_STRIDE 1 +#define REC_R_X x0 + +#define REC_PQ_DEFINE() v_t x0, y0, d0 +#define REC_PQ_STRIDE 1 +#define REC_PQ_X x0 +#define REC_PQ_Y y0 +#define REC_PQ_D d0 + +#define REC_PR_DEFINE() v_t x0, y0, d0 +#define REC_PR_STRIDE 1 +#define REC_PR_X x0 +#define REC_PR_Y y0 +#define REC_PR_D d0 + +#define REC_QR_DEFINE() v_t x0, y0, d0 +#define REC_QR_STRIDE 1 +#define REC_QR_X x0 +#define REC_QR_Y y0 +#define REC_QR_D d0 + +#define REC_PQR_DEFINE() v_t x0, y0, z0, d0, t0 +#define REC_PQR_STRIDE 1 +#define REC_PQR_X x0 +#define REC_PQR_Y y0 +#define REC_PQR_Z z0 +#define REC_PQR_D d0 +#define REC_PQR_XS d0 +#define REC_PQR_YS t0 + +#include "vdev_raidz_math_impl.h" + +/* + * If compiled with -O0, gcc doesn't do any stack frame coalescing + * and -Wframe-larger-than=1024 is triggered in debug mode. + * Starting with gcc 4.8, new opt level -Og is introduced for debugging, which + * does not trigger this warning. + */ +#pragma GCC diagnostic ignored "-Wframe-larger-than=" + +DEFINE_GEN_METHODS(scalar); +DEFINE_REC_METHODS(scalar); + +boolean_t +raidz_will_scalar_work(void) +{ + return (B_TRUE); /* always */ +} + +const raidz_impl_ops_t vdev_raidz_scalar_impl = { + .init = raidz_init_scalar, + .fini = NULL, + .gen = RAIDZ_GEN_METHODS(scalar), + .rec = RAIDZ_REC_METHODS(scalar), + .is_supported = &raidz_will_scalar_work, + .name = "scalar" +}; + +/* Powers of 2 in the RAID-Z Galois field. */ +const uint8_t vdev_raidz_pow2[256] __attribute__((aligned(256))) = { + 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, + 0x1d, 0x3a, 0x74, 0xe8, 0xcd, 0x87, 0x13, 0x26, + 0x4c, 0x98, 0x2d, 0x5a, 0xb4, 0x75, 0xea, 0xc9, + 0x8f, 0x03, 0x06, 0x0c, 0x18, 0x30, 0x60, 0xc0, + 0x9d, 0x27, 0x4e, 0x9c, 0x25, 0x4a, 0x94, 0x35, + 0x6a, 0xd4, 0xb5, 0x77, 0xee, 0xc1, 0x9f, 0x23, + 0x46, 0x8c, 0x05, 0x0a, 0x14, 0x28, 0x50, 0xa0, + 0x5d, 0xba, 0x69, 0xd2, 0xb9, 0x6f, 0xde, 0xa1, + 0x5f, 0xbe, 0x61, 0xc2, 0x99, 0x2f, 0x5e, 0xbc, + 0x65, 0xca, 0x89, 0x0f, 0x1e, 0x3c, 0x78, 0xf0, + 0xfd, 0xe7, 0xd3, 0xbb, 0x6b, 0xd6, 0xb1, 0x7f, + 0xfe, 0xe1, 0xdf, 0xa3, 0x5b, 0xb6, 0x71, 0xe2, + 0xd9, 0xaf, 0x43, 0x86, 0x11, 0x22, 0x44, 0x88, + 0x0d, 0x1a, 0x34, 0x68, 0xd0, 0xbd, 0x67, 0xce, + 0x81, 0x1f, 0x3e, 0x7c, 0xf8, 0xed, 0xc7, 0x93, + 0x3b, 0x76, 0xec, 0xc5, 0x97, 0x33, 0x66, 0xcc, + 0x85, 0x17, 0x2e, 0x5c, 0xb8, 0x6d, 0xda, 0xa9, + 0x4f, 0x9e, 0x21, 0x42, 0x84, 0x15, 0x2a, 0x54, + 0xa8, 0x4d, 0x9a, 0x29, 0x52, 0xa4, 0x55, 0xaa, + 0x49, 0x92, 0x39, 0x72, 0xe4, 0xd5, 0xb7, 0x73, + 0xe6, 0xd1, 0xbf, 0x63, 0xc6, 0x91, 0x3f, 0x7e, + 0xfc, 0xe5, 0xd7, 0xb3, 0x7b, 0xf6, 0xf1, 0xff, + 0xe3, 0xdb, 0xab, 0x4b, 0x96, 0x31, 0x62, 0xc4, + 0x95, 0x37, 0x6e, 0xdc, 0xa5, 0x57, 0xae, 0x41, + 0x82, 0x19, 0x32, 0x64, 0xc8, 0x8d, 0x07, 0x0e, + 0x1c, 0x38, 0x70, 0xe0, 0xdd, 0xa7, 0x53, 0xa6, + 0x51, 0xa2, 0x59, 0xb2, 0x79, 0xf2, 0xf9, 0xef, + 0xc3, 0x9b, 0x2b, 0x56, 0xac, 0x45, 0x8a, 0x09, + 0x12, 0x24, 0x48, 0x90, 0x3d, 0x7a, 0xf4, 0xf5, + 0xf7, 0xf3, 0xfb, 0xeb, 0xcb, 0x8b, 0x0b, 0x16, + 0x2c, 0x58, 0xb0, 0x7d, 0xfa, 0xe9, 0xcf, 0x83, + 0x1b, 0x36, 0x6c, 0xd8, 0xad, 0x47, 0x8e, 0x01 +}; + +/* Logs of 2 in the RAID-Z Galois field. */ +const uint8_t vdev_raidz_log2[256] __attribute__((aligned(256))) = { + 0x00, 0x00, 0x01, 0x19, 0x02, 0x32, 0x1a, 0xc6, + 0x03, 0xdf, 0x33, 0xee, 0x1b, 0x68, 0xc7, 0x4b, + 0x04, 0x64, 0xe0, 0x0e, 0x34, 0x8d, 0xef, 0x81, + 0x1c, 0xc1, 0x69, 0xf8, 0xc8, 0x08, 0x4c, 0x71, + 0x05, 0x8a, 0x65, 0x2f, 0xe1, 0x24, 0x0f, 0x21, + 0x35, 0x93, 0x8e, 0xda, 0xf0, 0x12, 0x82, 0x45, + 0x1d, 0xb5, 0xc2, 0x7d, 0x6a, 0x27, 0xf9, 0xb9, + 0xc9, 0x9a, 0x09, 0x78, 0x4d, 0xe4, 0x72, 0xa6, + 0x06, 0xbf, 0x8b, 0x62, 0x66, 0xdd, 0x30, 0xfd, + 0xe2, 0x98, 0x25, 0xb3, 0x10, 0x91, 0x22, 0x88, + 0x36, 0xd0, 0x94, 0xce, 0x8f, 0x96, 0xdb, 0xbd, + 0xf1, 0xd2, 0x13, 0x5c, 0x83, 0x38, 0x46, 0x40, + 0x1e, 0x42, 0xb6, 0xa3, 0xc3, 0x48, 0x7e, 0x6e, + 0x6b, 0x3a, 0x28, 0x54, 0xfa, 0x85, 0xba, 0x3d, + 0xca, 0x5e, 0x9b, 0x9f, 0x0a, 0x15, 0x79, 0x2b, + 0x4e, 0xd4, 0xe5, 0xac, 0x73, 0xf3, 0xa7, 0x57, + 0x07, 0x70, 0xc0, 0xf7, 0x8c, 0x80, 0x63, 0x0d, + 0x67, 0x4a, 0xde, 0xed, 0x31, 0xc5, 0xfe, 0x18, + 0xe3, 0xa5, 0x99, 0x77, 0x26, 0xb8, 0xb4, 0x7c, + 0x11, 0x44, 0x92, 0xd9, 0x23, 0x20, 0x89, 0x2e, + 0x37, 0x3f, 0xd1, 0x5b, 0x95, 0xbc, 0xcf, 0xcd, + 0x90, 0x87, 0x97, 0xb2, 0xdc, 0xfc, 0xbe, 0x61, + 0xf2, 0x56, 0xd3, 0xab, 0x14, 0x2a, 0x5d, 0x9e, + 0x84, 0x3c, 0x39, 0x53, 0x47, 0x6d, 0x41, 0xa2, + 0x1f, 0x2d, 0x43, 0xd8, 0xb7, 0x7b, 0xa4, 0x76, + 0xc4, 0x17, 0x49, 0xec, 0x7f, 0x0c, 0x6f, 0xf6, + 0x6c, 0xa1, 0x3b, 0x52, 0x29, 0x9d, 0x55, 0xaa, + 0xfb, 0x60, 0x86, 0xb1, 0xbb, 0xcc, 0x3e, 0x5a, + 0xcb, 0x59, 0x5f, 0xb0, 0x9c, 0xa9, 0xa0, 0x51, + 0x0b, 0xf5, 0x16, 0xeb, 0x7a, 0x75, 0x2c, 0xd7, + 0x4f, 0xae, 0xd5, 0xe9, 0xe6, 0xe7, 0xad, 0xe8, + 0x74, 0xd6, 0xf4, 0xea, 0xa8, 0x50, 0x58, 0xaf, +}; diff --git a/module/zfs/vdev_raidz_math_sse2.c b/module/zfs/vdev_raidz_math_sse2.c new file mode 100644 index 000000000000..1f8368418281 --- /dev/null +++ b/module/zfs/vdev_raidz_math_sse2.c @@ -0,0 +1,593 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. + */ + +#include + +#if defined(__x86_64) && defined(HAVE_SSE2) + +#include +#include + +#define __asm __asm__ __volatile__ + +#define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N +#define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) + +#define VR0_(REG, ...) "xmm"#REG +#define VR1_(_1, REG, ...) "xmm"#REG +#define VR2_(_1, _2, REG, ...) "xmm"#REG +#define VR3_(_1, _2, _3, REG, ...) "xmm"#REG +#define VR4_(_1, _2, _3, _4, REG, ...) "xmm"#REG +#define VR5_(_1, _2, _3, _4, _5, REG, ...) "xmm"#REG +#define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "xmm"#REG +#define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "xmm"#REG + +#define VR0(r...) VR0_(r, 1, 2, 3, 4, 5, 6) +#define VR1(r...) VR1_(r, 1, 2, 3, 4, 5, 6) +#define VR2(r...) VR2_(r, 1, 2, 3, 4, 5, 6) +#define VR3(r...) VR3_(r, 1, 2, 3, 4, 5, 6) +#define VR4(r...) VR4_(r, 1, 2, 3, 4, 5, 6) +#define VR5(r...) VR5_(r, 1, 2, 3, 4, 5, 6) +#define VR6(r...) VR6_(r, 1, 2, 3, 4, 5, 6) +#define VR7(r...) VR7_(r, 1, 2, 3, 4, 5, 6) + +#define ELEM_SIZE 16 + +typedef struct v { + uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); +} v_t; + +#define PREFETCHNTA(ptr, offset) {} +#define PREFETCH(ptr, offset) {} + +#define XOR_ACC(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ + "pxor 0x10(%[SRC]), %%" VR1(r) "\n" \ + "pxor 0x20(%[SRC]), %%" VR2(r) "\n" \ + "pxor 0x30(%[SRC]), %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ + "pxor 0x10(%[SRC]), %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 1: \ + __asm("pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + } \ +} + +#define XOR(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "pxor %" VR0(r) ", %" VR4(r) "\n" \ + "pxor %" VR1(r) ", %" VR5(r) "\n" \ + "pxor %" VR2(r) ", %" VR6(r) "\n" \ + "pxor %" VR3(r) ", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "pxor %" VR0(r) ", %" VR2(r) "\n" \ + "pxor %" VR1(r) ", %" VR3(r)); \ + break; \ + case 2: \ + __asm( \ + "pxor %" VR0(r) ", %" VR1(r)); \ + break; \ + } \ +} + +#define COPY(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "movdqa %" VR0(r) ", %" VR4(r) "\n" \ + "movdqa %" VR1(r) ", %" VR5(r) "\n" \ + "movdqa %" VR2(r) ", %" VR6(r) "\n" \ + "movdqa %" VR3(r) ", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "movdqa %" VR0(r) ", %" VR2(r) "\n" \ + "movdqa %" VR1(r) ", %" VR3(r)); \ + break; \ + case 2: \ + __asm( \ + "movdqa %" VR0(r) ", %" VR1(r)); \ + break; \ + } \ +} + +#define LOAD(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ + "movdqa 0x10(%[SRC]), %%" VR1(r) "\n" \ + "movdqa 0x20(%[SRC]), %%" VR2(r) "\n" \ + "movdqa 0x30(%[SRC]), %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ + "movdqa 0x10(%[SRC]), %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 1: \ + __asm( \ + "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + } \ +} + +#define STORE(dst, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ + "movdqa %%" VR1(r)", 0x10(%[DST])\n" \ + "movdqa %%" VR2(r)", 0x20(%[DST])\n" \ + "movdqa %%" VR3(r)", 0x30(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + case 2: \ + __asm( \ + "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ + "movdqa %%" VR1(r)", 0x10(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + case 1: \ + __asm( \ + "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + } \ +} + +#define MUL2_SETUP() \ +{ \ + __asm( \ + "movd %[mask], %%xmm15\n" \ + "pshufd $0x0, %%xmm15, %%xmm15\n" \ + : : [mask] "r" (0x1d1d1d1d)); \ +} + +#define _MUL2_x1(a0) \ +{ \ + __asm( \ + "pxor %xmm14, %xmm14\n" \ + "pcmpgtb %" a0", %xmm14\n" \ + "pand %xmm15, %xmm14\n" \ + "paddb %" a0", %" a0 "\n" \ + "pxor %xmm14, %" a0); \ +} + +#define _MUL2_x2(a0, a1) \ +{ \ + __asm( \ + "pxor %xmm14, %xmm14\n" \ + "pxor %xmm13, %xmm13\n" \ + "pcmpgtb %" a0", %xmm14\n" \ + "pcmpgtb %" a1", %xmm13\n" \ + "pand %xmm15, %xmm14\n" \ + "pand %xmm15, %xmm13\n" \ + "paddb %" a0", %" a0 "\n" \ + "paddb %" a1", %" a1 "\n" \ + "pxor %xmm14, %" a0 "\n" \ + "pxor %xmm13, %" a1); \ +} + +#define MUL2(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + _MUL2_x2(VR0(r), VR1(r)); \ + break; \ + case 1: \ + _MUL2_x1(VR0(r)); \ + break; \ + } \ +} + +#define MUL4(r...) \ +{ \ + MUL2(r); \ + MUL2(r); \ +} + +/* General multiplication by adding powers of two */ + +#define _MUL_PARAM(x, in, acc) \ +{ \ + if (x & 0x01) { COPY(in, acc); } else { XOR(acc, acc); } \ + if (x & 0xfe) { MUL2(in); } \ + if (x & 0x02) { XOR(in, acc); } \ + if (x & 0xfc) { MUL2(in); } \ + if (x & 0x04) { XOR(in, acc); } \ + if (x & 0xf8) { MUL2(in); } \ + if (x & 0x08) { XOR(in, acc); } \ + if (x & 0xf0) { MUL2(in); } \ + if (x & 0x10) { XOR(in, acc); } \ + if (x & 0xe0) { MUL2(in); } \ + if (x & 0x20) { XOR(in, acc); } \ + if (x & 0xc0) { MUL2(in); } \ + if (x & 0x40) { XOR(in, acc); } \ + if (x & 0x80) { MUL2(in); XOR(in, acc); } \ +} + +#define _mul_x1_in 9 +#define _mul_x1_acc 11 + +#define MUL_x1_DEFINE(x) \ +static void \ +mul_x1_ ## x(void) { _MUL_PARAM(x, _mul_x1_in, _mul_x1_acc); } + +#define _mul_x2_in 9, 10 +#define _mul_x2_acc 11, 12 + +#define MUL_x2_DEFINE(x) \ +static void \ +mul_x2_ ## x(void) { _MUL_PARAM(x, _mul_x2_in, _mul_x2_acc); } + +MUL_x1_DEFINE(0); MUL_x1_DEFINE(1); MUL_x1_DEFINE(2); MUL_x1_DEFINE(3); +MUL_x1_DEFINE(4); MUL_x1_DEFINE(5); MUL_x1_DEFINE(6); MUL_x1_DEFINE(7); +MUL_x1_DEFINE(8); MUL_x1_DEFINE(9); MUL_x1_DEFINE(10); MUL_x1_DEFINE(11); +MUL_x1_DEFINE(12); MUL_x1_DEFINE(13); MUL_x1_DEFINE(14); MUL_x1_DEFINE(15); +MUL_x1_DEFINE(16); MUL_x1_DEFINE(17); MUL_x1_DEFINE(18); MUL_x1_DEFINE(19); +MUL_x1_DEFINE(20); MUL_x1_DEFINE(21); MUL_x1_DEFINE(22); MUL_x1_DEFINE(23); +MUL_x1_DEFINE(24); MUL_x1_DEFINE(25); MUL_x1_DEFINE(26); MUL_x1_DEFINE(27); +MUL_x1_DEFINE(28); MUL_x1_DEFINE(29); MUL_x1_DEFINE(30); MUL_x1_DEFINE(31); +MUL_x1_DEFINE(32); MUL_x1_DEFINE(33); MUL_x1_DEFINE(34); MUL_x1_DEFINE(35); +MUL_x1_DEFINE(36); MUL_x1_DEFINE(37); MUL_x1_DEFINE(38); MUL_x1_DEFINE(39); +MUL_x1_DEFINE(40); MUL_x1_DEFINE(41); MUL_x1_DEFINE(42); MUL_x1_DEFINE(43); +MUL_x1_DEFINE(44); MUL_x1_DEFINE(45); MUL_x1_DEFINE(46); MUL_x1_DEFINE(47); +MUL_x1_DEFINE(48); MUL_x1_DEFINE(49); MUL_x1_DEFINE(50); MUL_x1_DEFINE(51); +MUL_x1_DEFINE(52); MUL_x1_DEFINE(53); MUL_x1_DEFINE(54); MUL_x1_DEFINE(55); +MUL_x1_DEFINE(56); MUL_x1_DEFINE(57); MUL_x1_DEFINE(58); MUL_x1_DEFINE(59); +MUL_x1_DEFINE(60); MUL_x1_DEFINE(61); MUL_x1_DEFINE(62); MUL_x1_DEFINE(63); +MUL_x1_DEFINE(64); MUL_x1_DEFINE(65); MUL_x1_DEFINE(66); MUL_x1_DEFINE(67); +MUL_x1_DEFINE(68); MUL_x1_DEFINE(69); MUL_x1_DEFINE(70); MUL_x1_DEFINE(71); +MUL_x1_DEFINE(72); MUL_x1_DEFINE(73); MUL_x1_DEFINE(74); MUL_x1_DEFINE(75); +MUL_x1_DEFINE(76); MUL_x1_DEFINE(77); MUL_x1_DEFINE(78); MUL_x1_DEFINE(79); +MUL_x1_DEFINE(80); MUL_x1_DEFINE(81); MUL_x1_DEFINE(82); MUL_x1_DEFINE(83); +MUL_x1_DEFINE(84); MUL_x1_DEFINE(85); MUL_x1_DEFINE(86); MUL_x1_DEFINE(87); +MUL_x1_DEFINE(88); MUL_x1_DEFINE(89); MUL_x1_DEFINE(90); MUL_x1_DEFINE(91); +MUL_x1_DEFINE(92); MUL_x1_DEFINE(93); MUL_x1_DEFINE(94); MUL_x1_DEFINE(95); +MUL_x1_DEFINE(96); MUL_x1_DEFINE(97); MUL_x1_DEFINE(98); MUL_x1_DEFINE(99); +MUL_x1_DEFINE(100); MUL_x1_DEFINE(101); MUL_x1_DEFINE(102); MUL_x1_DEFINE(103); +MUL_x1_DEFINE(104); MUL_x1_DEFINE(105); MUL_x1_DEFINE(106); MUL_x1_DEFINE(107); +MUL_x1_DEFINE(108); MUL_x1_DEFINE(109); MUL_x1_DEFINE(110); MUL_x1_DEFINE(111); +MUL_x1_DEFINE(112); MUL_x1_DEFINE(113); MUL_x1_DEFINE(114); MUL_x1_DEFINE(115); +MUL_x1_DEFINE(116); MUL_x1_DEFINE(117); MUL_x1_DEFINE(118); MUL_x1_DEFINE(119); +MUL_x1_DEFINE(120); MUL_x1_DEFINE(121); MUL_x1_DEFINE(122); MUL_x1_DEFINE(123); +MUL_x1_DEFINE(124); MUL_x1_DEFINE(125); MUL_x1_DEFINE(126); MUL_x1_DEFINE(127); +MUL_x1_DEFINE(128); MUL_x1_DEFINE(129); MUL_x1_DEFINE(130); MUL_x1_DEFINE(131); +MUL_x1_DEFINE(132); MUL_x1_DEFINE(133); MUL_x1_DEFINE(134); MUL_x1_DEFINE(135); +MUL_x1_DEFINE(136); MUL_x1_DEFINE(137); MUL_x1_DEFINE(138); MUL_x1_DEFINE(139); +MUL_x1_DEFINE(140); MUL_x1_DEFINE(141); MUL_x1_DEFINE(142); MUL_x1_DEFINE(143); +MUL_x1_DEFINE(144); MUL_x1_DEFINE(145); MUL_x1_DEFINE(146); MUL_x1_DEFINE(147); +MUL_x1_DEFINE(148); MUL_x1_DEFINE(149); MUL_x1_DEFINE(150); MUL_x1_DEFINE(151); +MUL_x1_DEFINE(152); MUL_x1_DEFINE(153); MUL_x1_DEFINE(154); MUL_x1_DEFINE(155); +MUL_x1_DEFINE(156); MUL_x1_DEFINE(157); MUL_x1_DEFINE(158); MUL_x1_DEFINE(159); +MUL_x1_DEFINE(160); MUL_x1_DEFINE(161); MUL_x1_DEFINE(162); MUL_x1_DEFINE(163); +MUL_x1_DEFINE(164); MUL_x1_DEFINE(165); MUL_x1_DEFINE(166); MUL_x1_DEFINE(167); +MUL_x1_DEFINE(168); MUL_x1_DEFINE(169); MUL_x1_DEFINE(170); MUL_x1_DEFINE(171); +MUL_x1_DEFINE(172); MUL_x1_DEFINE(173); MUL_x1_DEFINE(174); MUL_x1_DEFINE(175); +MUL_x1_DEFINE(176); MUL_x1_DEFINE(177); MUL_x1_DEFINE(178); MUL_x1_DEFINE(179); +MUL_x1_DEFINE(180); MUL_x1_DEFINE(181); MUL_x1_DEFINE(182); MUL_x1_DEFINE(183); +MUL_x1_DEFINE(184); MUL_x1_DEFINE(185); MUL_x1_DEFINE(186); MUL_x1_DEFINE(187); +MUL_x1_DEFINE(188); MUL_x1_DEFINE(189); MUL_x1_DEFINE(190); MUL_x1_DEFINE(191); +MUL_x1_DEFINE(192); MUL_x1_DEFINE(193); MUL_x1_DEFINE(194); MUL_x1_DEFINE(195); +MUL_x1_DEFINE(196); MUL_x1_DEFINE(197); MUL_x1_DEFINE(198); MUL_x1_DEFINE(199); +MUL_x1_DEFINE(200); MUL_x1_DEFINE(201); MUL_x1_DEFINE(202); MUL_x1_DEFINE(203); +MUL_x1_DEFINE(204); MUL_x1_DEFINE(205); MUL_x1_DEFINE(206); MUL_x1_DEFINE(207); +MUL_x1_DEFINE(208); MUL_x1_DEFINE(209); MUL_x1_DEFINE(210); MUL_x1_DEFINE(211); +MUL_x1_DEFINE(212); MUL_x1_DEFINE(213); MUL_x1_DEFINE(214); MUL_x1_DEFINE(215); +MUL_x1_DEFINE(216); MUL_x1_DEFINE(217); MUL_x1_DEFINE(218); MUL_x1_DEFINE(219); +MUL_x1_DEFINE(220); MUL_x1_DEFINE(221); MUL_x1_DEFINE(222); MUL_x1_DEFINE(223); +MUL_x1_DEFINE(224); MUL_x1_DEFINE(225); MUL_x1_DEFINE(226); MUL_x1_DEFINE(227); +MUL_x1_DEFINE(228); MUL_x1_DEFINE(229); MUL_x1_DEFINE(230); MUL_x1_DEFINE(231); +MUL_x1_DEFINE(232); MUL_x1_DEFINE(233); MUL_x1_DEFINE(234); MUL_x1_DEFINE(235); +MUL_x1_DEFINE(236); MUL_x1_DEFINE(237); MUL_x1_DEFINE(238); MUL_x1_DEFINE(239); +MUL_x1_DEFINE(240); MUL_x1_DEFINE(241); MUL_x1_DEFINE(242); MUL_x1_DEFINE(243); +MUL_x1_DEFINE(244); MUL_x1_DEFINE(245); MUL_x1_DEFINE(246); MUL_x1_DEFINE(247); +MUL_x1_DEFINE(248); MUL_x1_DEFINE(249); MUL_x1_DEFINE(250); MUL_x1_DEFINE(251); +MUL_x1_DEFINE(252); MUL_x1_DEFINE(253); MUL_x1_DEFINE(254); MUL_x1_DEFINE(255); + +MUL_x2_DEFINE(0); MUL_x2_DEFINE(1); MUL_x2_DEFINE(2); MUL_x2_DEFINE(3); +MUL_x2_DEFINE(4); MUL_x2_DEFINE(5); MUL_x2_DEFINE(6); MUL_x2_DEFINE(7); +MUL_x2_DEFINE(8); MUL_x2_DEFINE(9); MUL_x2_DEFINE(10); MUL_x2_DEFINE(11); +MUL_x2_DEFINE(12); MUL_x2_DEFINE(13); MUL_x2_DEFINE(14); MUL_x2_DEFINE(15); +MUL_x2_DEFINE(16); MUL_x2_DEFINE(17); MUL_x2_DEFINE(18); MUL_x2_DEFINE(19); +MUL_x2_DEFINE(20); MUL_x2_DEFINE(21); MUL_x2_DEFINE(22); MUL_x2_DEFINE(23); +MUL_x2_DEFINE(24); MUL_x2_DEFINE(25); MUL_x2_DEFINE(26); MUL_x2_DEFINE(27); +MUL_x2_DEFINE(28); MUL_x2_DEFINE(29); MUL_x2_DEFINE(30); MUL_x2_DEFINE(31); +MUL_x2_DEFINE(32); MUL_x2_DEFINE(33); MUL_x2_DEFINE(34); MUL_x2_DEFINE(35); +MUL_x2_DEFINE(36); MUL_x2_DEFINE(37); MUL_x2_DEFINE(38); MUL_x2_DEFINE(39); +MUL_x2_DEFINE(40); MUL_x2_DEFINE(41); MUL_x2_DEFINE(42); MUL_x2_DEFINE(43); +MUL_x2_DEFINE(44); MUL_x2_DEFINE(45); MUL_x2_DEFINE(46); MUL_x2_DEFINE(47); +MUL_x2_DEFINE(48); MUL_x2_DEFINE(49); MUL_x2_DEFINE(50); MUL_x2_DEFINE(51); +MUL_x2_DEFINE(52); MUL_x2_DEFINE(53); MUL_x2_DEFINE(54); MUL_x2_DEFINE(55); +MUL_x2_DEFINE(56); MUL_x2_DEFINE(57); MUL_x2_DEFINE(58); MUL_x2_DEFINE(59); +MUL_x2_DEFINE(60); MUL_x2_DEFINE(61); MUL_x2_DEFINE(62); MUL_x2_DEFINE(63); +MUL_x2_DEFINE(64); MUL_x2_DEFINE(65); MUL_x2_DEFINE(66); MUL_x2_DEFINE(67); +MUL_x2_DEFINE(68); MUL_x2_DEFINE(69); MUL_x2_DEFINE(70); MUL_x2_DEFINE(71); +MUL_x2_DEFINE(72); MUL_x2_DEFINE(73); MUL_x2_DEFINE(74); MUL_x2_DEFINE(75); +MUL_x2_DEFINE(76); MUL_x2_DEFINE(77); MUL_x2_DEFINE(78); MUL_x2_DEFINE(79); +MUL_x2_DEFINE(80); MUL_x2_DEFINE(81); MUL_x2_DEFINE(82); MUL_x2_DEFINE(83); +MUL_x2_DEFINE(84); MUL_x2_DEFINE(85); MUL_x2_DEFINE(86); MUL_x2_DEFINE(87); +MUL_x2_DEFINE(88); MUL_x2_DEFINE(89); MUL_x2_DEFINE(90); MUL_x2_DEFINE(91); +MUL_x2_DEFINE(92); MUL_x2_DEFINE(93); MUL_x2_DEFINE(94); MUL_x2_DEFINE(95); +MUL_x2_DEFINE(96); MUL_x2_DEFINE(97); MUL_x2_DEFINE(98); MUL_x2_DEFINE(99); +MUL_x2_DEFINE(100); MUL_x2_DEFINE(101); MUL_x2_DEFINE(102); MUL_x2_DEFINE(103); +MUL_x2_DEFINE(104); MUL_x2_DEFINE(105); MUL_x2_DEFINE(106); MUL_x2_DEFINE(107); +MUL_x2_DEFINE(108); MUL_x2_DEFINE(109); MUL_x2_DEFINE(110); MUL_x2_DEFINE(111); +MUL_x2_DEFINE(112); MUL_x2_DEFINE(113); MUL_x2_DEFINE(114); MUL_x2_DEFINE(115); +MUL_x2_DEFINE(116); MUL_x2_DEFINE(117); MUL_x2_DEFINE(118); MUL_x2_DEFINE(119); +MUL_x2_DEFINE(120); MUL_x2_DEFINE(121); MUL_x2_DEFINE(122); MUL_x2_DEFINE(123); +MUL_x2_DEFINE(124); MUL_x2_DEFINE(125); MUL_x2_DEFINE(126); MUL_x2_DEFINE(127); +MUL_x2_DEFINE(128); MUL_x2_DEFINE(129); MUL_x2_DEFINE(130); MUL_x2_DEFINE(131); +MUL_x2_DEFINE(132); MUL_x2_DEFINE(133); MUL_x2_DEFINE(134); MUL_x2_DEFINE(135); +MUL_x2_DEFINE(136); MUL_x2_DEFINE(137); MUL_x2_DEFINE(138); MUL_x2_DEFINE(139); +MUL_x2_DEFINE(140); MUL_x2_DEFINE(141); MUL_x2_DEFINE(142); MUL_x2_DEFINE(143); +MUL_x2_DEFINE(144); MUL_x2_DEFINE(145); MUL_x2_DEFINE(146); MUL_x2_DEFINE(147); +MUL_x2_DEFINE(148); MUL_x2_DEFINE(149); MUL_x2_DEFINE(150); MUL_x2_DEFINE(151); +MUL_x2_DEFINE(152); MUL_x2_DEFINE(153); MUL_x2_DEFINE(154); MUL_x2_DEFINE(155); +MUL_x2_DEFINE(156); MUL_x2_DEFINE(157); MUL_x2_DEFINE(158); MUL_x2_DEFINE(159); +MUL_x2_DEFINE(160); MUL_x2_DEFINE(161); MUL_x2_DEFINE(162); MUL_x2_DEFINE(163); +MUL_x2_DEFINE(164); MUL_x2_DEFINE(165); MUL_x2_DEFINE(166); MUL_x2_DEFINE(167); +MUL_x2_DEFINE(168); MUL_x2_DEFINE(169); MUL_x2_DEFINE(170); MUL_x2_DEFINE(171); +MUL_x2_DEFINE(172); MUL_x2_DEFINE(173); MUL_x2_DEFINE(174); MUL_x2_DEFINE(175); +MUL_x2_DEFINE(176); MUL_x2_DEFINE(177); MUL_x2_DEFINE(178); MUL_x2_DEFINE(179); +MUL_x2_DEFINE(180); MUL_x2_DEFINE(181); MUL_x2_DEFINE(182); MUL_x2_DEFINE(183); +MUL_x2_DEFINE(184); MUL_x2_DEFINE(185); MUL_x2_DEFINE(186); MUL_x2_DEFINE(187); +MUL_x2_DEFINE(188); MUL_x2_DEFINE(189); MUL_x2_DEFINE(190); MUL_x2_DEFINE(191); +MUL_x2_DEFINE(192); MUL_x2_DEFINE(193); MUL_x2_DEFINE(194); MUL_x2_DEFINE(195); +MUL_x2_DEFINE(196); MUL_x2_DEFINE(197); MUL_x2_DEFINE(198); MUL_x2_DEFINE(199); +MUL_x2_DEFINE(200); MUL_x2_DEFINE(201); MUL_x2_DEFINE(202); MUL_x2_DEFINE(203); +MUL_x2_DEFINE(204); MUL_x2_DEFINE(205); MUL_x2_DEFINE(206); MUL_x2_DEFINE(207); +MUL_x2_DEFINE(208); MUL_x2_DEFINE(209); MUL_x2_DEFINE(210); MUL_x2_DEFINE(211); +MUL_x2_DEFINE(212); MUL_x2_DEFINE(213); MUL_x2_DEFINE(214); MUL_x2_DEFINE(215); +MUL_x2_DEFINE(216); MUL_x2_DEFINE(217); MUL_x2_DEFINE(218); MUL_x2_DEFINE(219); +MUL_x2_DEFINE(220); MUL_x2_DEFINE(221); MUL_x2_DEFINE(222); MUL_x2_DEFINE(223); +MUL_x2_DEFINE(224); MUL_x2_DEFINE(225); MUL_x2_DEFINE(226); MUL_x2_DEFINE(227); +MUL_x2_DEFINE(228); MUL_x2_DEFINE(229); MUL_x2_DEFINE(230); MUL_x2_DEFINE(231); +MUL_x2_DEFINE(232); MUL_x2_DEFINE(233); MUL_x2_DEFINE(234); MUL_x2_DEFINE(235); +MUL_x2_DEFINE(236); MUL_x2_DEFINE(237); MUL_x2_DEFINE(238); MUL_x2_DEFINE(239); +MUL_x2_DEFINE(240); MUL_x2_DEFINE(241); MUL_x2_DEFINE(242); MUL_x2_DEFINE(243); +MUL_x2_DEFINE(244); MUL_x2_DEFINE(245); MUL_x2_DEFINE(246); MUL_x2_DEFINE(247); +MUL_x2_DEFINE(248); MUL_x2_DEFINE(249); MUL_x2_DEFINE(250); MUL_x2_DEFINE(251); +MUL_x2_DEFINE(252); MUL_x2_DEFINE(253); MUL_x2_DEFINE(254); MUL_x2_DEFINE(255); + + + +typedef void (*mul_fn_ptr_t)(void); + +static const mul_fn_ptr_t __attribute__((aligned(256))) +gf_x1_mul_fns[256] = { + mul_x1_0, mul_x1_1, mul_x1_2, mul_x1_3, mul_x1_4, mul_x1_5, + mul_x1_6, mul_x1_7, mul_x1_8, mul_x1_9, mul_x1_10, mul_x1_11, + mul_x1_12, mul_x1_13, mul_x1_14, mul_x1_15, mul_x1_16, mul_x1_17, + mul_x1_18, mul_x1_19, mul_x1_20, mul_x1_21, mul_x1_22, mul_x1_23, + mul_x1_24, mul_x1_25, mul_x1_26, mul_x1_27, mul_x1_28, mul_x1_29, + mul_x1_30, mul_x1_31, mul_x1_32, mul_x1_33, mul_x1_34, mul_x1_35, + mul_x1_36, mul_x1_37, mul_x1_38, mul_x1_39, mul_x1_40, mul_x1_41, + mul_x1_42, mul_x1_43, mul_x1_44, mul_x1_45, mul_x1_46, mul_x1_47, + mul_x1_48, mul_x1_49, mul_x1_50, mul_x1_51, mul_x1_52, mul_x1_53, + mul_x1_54, mul_x1_55, mul_x1_56, mul_x1_57, mul_x1_58, mul_x1_59, + mul_x1_60, mul_x1_61, mul_x1_62, mul_x1_63, mul_x1_64, mul_x1_65, + mul_x1_66, mul_x1_67, mul_x1_68, mul_x1_69, mul_x1_70, mul_x1_71, + mul_x1_72, mul_x1_73, mul_x1_74, mul_x1_75, mul_x1_76, mul_x1_77, + mul_x1_78, mul_x1_79, mul_x1_80, mul_x1_81, mul_x1_82, mul_x1_83, + mul_x1_84, mul_x1_85, mul_x1_86, mul_x1_87, mul_x1_88, mul_x1_89, + mul_x1_90, mul_x1_91, mul_x1_92, mul_x1_93, mul_x1_94, mul_x1_95, + mul_x1_96, mul_x1_97, mul_x1_98, mul_x1_99, mul_x1_100, mul_x1_101, + mul_x1_102, mul_x1_103, mul_x1_104, mul_x1_105, mul_x1_106, mul_x1_107, + mul_x1_108, mul_x1_109, mul_x1_110, mul_x1_111, mul_x1_112, mul_x1_113, + mul_x1_114, mul_x1_115, mul_x1_116, mul_x1_117, mul_x1_118, mul_x1_119, + mul_x1_120, mul_x1_121, mul_x1_122, mul_x1_123, mul_x1_124, mul_x1_125, + mul_x1_126, mul_x1_127, mul_x1_128, mul_x1_129, mul_x1_130, mul_x1_131, + mul_x1_132, mul_x1_133, mul_x1_134, mul_x1_135, mul_x1_136, mul_x1_137, + mul_x1_138, mul_x1_139, mul_x1_140, mul_x1_141, mul_x1_142, mul_x1_143, + mul_x1_144, mul_x1_145, mul_x1_146, mul_x1_147, mul_x1_148, mul_x1_149, + mul_x1_150, mul_x1_151, mul_x1_152, mul_x1_153, mul_x1_154, mul_x1_155, + mul_x1_156, mul_x1_157, mul_x1_158, mul_x1_159, mul_x1_160, mul_x1_161, + mul_x1_162, mul_x1_163, mul_x1_164, mul_x1_165, mul_x1_166, mul_x1_167, + mul_x1_168, mul_x1_169, mul_x1_170, mul_x1_171, mul_x1_172, mul_x1_173, + mul_x1_174, mul_x1_175, mul_x1_176, mul_x1_177, mul_x1_178, mul_x1_179, + mul_x1_180, mul_x1_181, mul_x1_182, mul_x1_183, mul_x1_184, mul_x1_185, + mul_x1_186, mul_x1_187, mul_x1_188, mul_x1_189, mul_x1_190, mul_x1_191, + mul_x1_192, mul_x1_193, mul_x1_194, mul_x1_195, mul_x1_196, mul_x1_197, + mul_x1_198, mul_x1_199, mul_x1_200, mul_x1_201, mul_x1_202, mul_x1_203, + mul_x1_204, mul_x1_205, mul_x1_206, mul_x1_207, mul_x1_208, mul_x1_209, + mul_x1_210, mul_x1_211, mul_x1_212, mul_x1_213, mul_x1_214, mul_x1_215, + mul_x1_216, mul_x1_217, mul_x1_218, mul_x1_219, mul_x1_220, mul_x1_221, + mul_x1_222, mul_x1_223, mul_x1_224, mul_x1_225, mul_x1_226, mul_x1_227, + mul_x1_228, mul_x1_229, mul_x1_230, mul_x1_231, mul_x1_232, mul_x1_233, + mul_x1_234, mul_x1_235, mul_x1_236, mul_x1_237, mul_x1_238, mul_x1_239, + mul_x1_240, mul_x1_241, mul_x1_242, mul_x1_243, mul_x1_244, mul_x1_245, + mul_x1_246, mul_x1_247, mul_x1_248, mul_x1_249, mul_x1_250, mul_x1_251, + mul_x1_252, mul_x1_253, mul_x1_254, mul_x1_255 +}; + +static const mul_fn_ptr_t __attribute__((aligned(256))) +gf_x2_mul_fns[256] = { + mul_x2_0, mul_x2_1, mul_x2_2, mul_x2_3, mul_x2_4, mul_x2_5, + mul_x2_6, mul_x2_7, mul_x2_8, mul_x2_9, mul_x2_10, mul_x2_11, + mul_x2_12, mul_x2_13, mul_x2_14, mul_x2_15, mul_x2_16, mul_x2_17, + mul_x2_18, mul_x2_19, mul_x2_20, mul_x2_21, mul_x2_22, mul_x2_23, + mul_x2_24, mul_x2_25, mul_x2_26, mul_x2_27, mul_x2_28, mul_x2_29, + mul_x2_30, mul_x2_31, mul_x2_32, mul_x2_33, mul_x2_34, mul_x2_35, + mul_x2_36, mul_x2_37, mul_x2_38, mul_x2_39, mul_x2_40, mul_x2_41, + mul_x2_42, mul_x2_43, mul_x2_44, mul_x2_45, mul_x2_46, mul_x2_47, + mul_x2_48, mul_x2_49, mul_x2_50, mul_x2_51, mul_x2_52, mul_x2_53, + mul_x2_54, mul_x2_55, mul_x2_56, mul_x2_57, mul_x2_58, mul_x2_59, + mul_x2_60, mul_x2_61, mul_x2_62, mul_x2_63, mul_x2_64, mul_x2_65, + mul_x2_66, mul_x2_67, mul_x2_68, mul_x2_69, mul_x2_70, mul_x2_71, + mul_x2_72, mul_x2_73, mul_x2_74, mul_x2_75, mul_x2_76, mul_x2_77, + mul_x2_78, mul_x2_79, mul_x2_80, mul_x2_81, mul_x2_82, mul_x2_83, + mul_x2_84, mul_x2_85, mul_x2_86, mul_x2_87, mul_x2_88, mul_x2_89, + mul_x2_90, mul_x2_91, mul_x2_92, mul_x2_93, mul_x2_94, mul_x2_95, + mul_x2_96, mul_x2_97, mul_x2_98, mul_x2_99, mul_x2_100, mul_x2_101, + mul_x2_102, mul_x2_103, mul_x2_104, mul_x2_105, mul_x2_106, mul_x2_107, + mul_x2_108, mul_x2_109, mul_x2_110, mul_x2_111, mul_x2_112, mul_x2_113, + mul_x2_114, mul_x2_115, mul_x2_116, mul_x2_117, mul_x2_118, mul_x2_119, + mul_x2_120, mul_x2_121, mul_x2_122, mul_x2_123, mul_x2_124, mul_x2_125, + mul_x2_126, mul_x2_127, mul_x2_128, mul_x2_129, mul_x2_130, mul_x2_131, + mul_x2_132, mul_x2_133, mul_x2_134, mul_x2_135, mul_x2_136, mul_x2_137, + mul_x2_138, mul_x2_139, mul_x2_140, mul_x2_141, mul_x2_142, mul_x2_143, + mul_x2_144, mul_x2_145, mul_x2_146, mul_x2_147, mul_x2_148, mul_x2_149, + mul_x2_150, mul_x2_151, mul_x2_152, mul_x2_153, mul_x2_154, mul_x2_155, + mul_x2_156, mul_x2_157, mul_x2_158, mul_x2_159, mul_x2_160, mul_x2_161, + mul_x2_162, mul_x2_163, mul_x2_164, mul_x2_165, mul_x2_166, mul_x2_167, + mul_x2_168, mul_x2_169, mul_x2_170, mul_x2_171, mul_x2_172, mul_x2_173, + mul_x2_174, mul_x2_175, mul_x2_176, mul_x2_177, mul_x2_178, mul_x2_179, + mul_x2_180, mul_x2_181, mul_x2_182, mul_x2_183, mul_x2_184, mul_x2_185, + mul_x2_186, mul_x2_187, mul_x2_188, mul_x2_189, mul_x2_190, mul_x2_191, + mul_x2_192, mul_x2_193, mul_x2_194, mul_x2_195, mul_x2_196, mul_x2_197, + mul_x2_198, mul_x2_199, mul_x2_200, mul_x2_201, mul_x2_202, mul_x2_203, + mul_x2_204, mul_x2_205, mul_x2_206, mul_x2_207, mul_x2_208, mul_x2_209, + mul_x2_210, mul_x2_211, mul_x2_212, mul_x2_213, mul_x2_214, mul_x2_215, + mul_x2_216, mul_x2_217, mul_x2_218, mul_x2_219, mul_x2_220, mul_x2_221, + mul_x2_222, mul_x2_223, mul_x2_224, mul_x2_225, mul_x2_226, mul_x2_227, + mul_x2_228, mul_x2_229, mul_x2_230, mul_x2_231, mul_x2_232, mul_x2_233, + mul_x2_234, mul_x2_235, mul_x2_236, mul_x2_237, mul_x2_238, mul_x2_239, + mul_x2_240, mul_x2_241, mul_x2_242, mul_x2_243, mul_x2_244, mul_x2_245, + mul_x2_246, mul_x2_247, mul_x2_248, mul_x2_249, mul_x2_250, mul_x2_251, + mul_x2_252, mul_x2_253, mul_x2_254, mul_x2_255 +}; + +#define MUL(c, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + COPY(r, _mul_x2_in); \ + gf_x2_mul_fns[c](); \ + COPY(_mul_x2_acc, r); \ + break; \ + case 1: \ + COPY(r, _mul_x1_in); \ + gf_x1_mul_fns[c](); \ + COPY(_mul_x1_acc, r); \ + break; \ + } \ +} + + +#define raidz_math_begin() kfpu_begin() +#define raidz_math_end() kfpu_end() + +#define GEN_P_DEFINE() {} +#define GEN_P_STRIDE 4 +#define GEN_P_P 0, 1, 2, 3 + +#define GEN_PQ_DEFINE() {} +#define GEN_PQ_STRIDE 2 +#define GEN_PQ_D 0, 1 +#define GEN_PQ_P 2, 3 +#define GEN_PQ_Q 4, 5 + +#define GEN_PQR_DEFINE() {} +#define GEN_PQR_STRIDE 2 +#define GEN_PQR_D 0, 1 +#define GEN_PQR_P 2, 3 +#define GEN_PQR_Q 4, 5 +#define GEN_PQR_R 6, 7 + +#define REC_P_DEFINE() {} +#define REC_P_STRIDE 4 +#define REC_P_X 0, 1, 2, 3 + +#define REC_Q_DEFINE() {} +#define REC_Q_STRIDE 2 +#define REC_Q_X 0, 1 + +#define REC_R_DEFINE() {} +#define REC_R_STRIDE 2 +#define REC_R_X 0, 1 + +#define REC_PQ_DEFINE() {} +#define REC_PQ_STRIDE 2 +#define REC_PQ_X 0, 1 +#define REC_PQ_Y 2, 3 +#define REC_PQ_D 4, 5 + +#define REC_PR_DEFINE() {} +#define REC_PR_STRIDE 2 +#define REC_PR_X 0, 1 +#define REC_PR_Y 2, 3 +#define REC_PR_D 4, 5 + +#define REC_QR_DEFINE() {} +#define REC_QR_STRIDE 2 +#define REC_QR_X 0, 1 +#define REC_QR_Y 2, 3 +#define REC_QR_D 4, 5 + +#define REC_PQR_DEFINE() {} +#define REC_PQR_STRIDE 1 +#define REC_PQR_X 0 +#define REC_PQR_Y 1 +#define REC_PQR_Z 2 +#define REC_PQR_D 3 +#define REC_PQR_XS 4 +#define REC_PQR_YS 5 + + +#include +#include "vdev_raidz_math_impl.h" + +DEFINE_GEN_METHODS(sse2); +DEFINE_REC_METHODS(sse2); + +static boolean_t +raidz_will_sse2_work(void) +{ + return (zfs_sse_available() && zfs_sse2_available()); +} + +const raidz_impl_ops_t vdev_raidz_sse2_impl = { + .init = NULL, + .fini = NULL, + .gen = RAIDZ_GEN_METHODS(sse2), + .rec = RAIDZ_REC_METHODS(sse2), + .is_supported = &raidz_will_sse2_work, + .name = "sse2" +}; + +#endif /* defined(__x86_64) && defined(HAVE_SSE2) */ diff --git a/module/zfs/vdev_raidz_math_ssse3.c b/module/zfs/vdev_raidz_math_ssse3.c new file mode 100644 index 000000000000..982e27efc487 --- /dev/null +++ b/module/zfs/vdev_raidz_math_ssse3.c @@ -0,0 +1,2456 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright (C) 2016 Gvozden Nešković. All rights reserved. + */ + +#include + +#if defined(__x86_64) && defined(HAVE_SSSE3) + +#include +#include + +#define __asm __asm__ __volatile__ + +#define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N +#define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) + +#define VR0_(REG, ...) "xmm"#REG +#define VR1_(_1, REG, ...) "xmm"#REG +#define VR2_(_1, _2, REG, ...) "xmm"#REG +#define VR3_(_1, _2, _3, REG, ...) "xmm"#REG +#define VR4_(_1, _2, _3, _4, REG, ...) "xmm"#REG +#define VR5_(_1, _2, _3, _4, _5, REG, ...) "xmm"#REG +#define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "xmm"#REG +#define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "xmm"#REG + +#define VR0(r...) VR0_(r) +#define VR1(r...) VR1_(r) +#define VR2(r...) VR2_(r, 1) +#define VR3(r...) VR3_(r, 1, 2) +#define VR4(r...) VR4_(r, 1, 2) +#define VR5(r...) VR5_(r, 1, 2, 3) +#define VR6(r...) VR6_(r, 1, 2, 3, 4) +#define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) + +#define R_01(REG1, REG2, ...) REG1, REG2 +#define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 +#define R_23(REG...) _R_23(REG, 1, 2, 3) + +#define ASM_BUG() ASSERT(0) + +const uint8_t gf_clmul_mod_lt[4*256][16]; + +#define ELEM_SIZE 16 + +typedef struct v { + uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); +} v_t; + +#define PREFETCHNTA(ptr, offset) \ +{ \ + __asm( \ + "prefetchnta " #offset "(%[MEM])\n" \ + : : [MEM] "r" (ptr)); \ +} + +#define PREFETCH(ptr, offset) \ +{ \ + __asm( \ + "prefetcht0 " #offset "(%[MEM])\n" \ + : : [MEM] "r" (ptr)); \ +} + +#define XOR_ACC(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ + "pxor 0x10(%[SRC]), %%" VR1(r) "\n" \ + "pxor 0x20(%[SRC]), %%" VR2(r) "\n" \ + "pxor 0x30(%[SRC]), %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ + "pxor 0x10(%[SRC]), %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define XOR(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "pxor %" VR0(r) ", %" VR4(r) "\n" \ + "pxor %" VR1(r) ", %" VR5(r) "\n" \ + "pxor %" VR2(r) ", %" VR6(r) "\n" \ + "pxor %" VR3(r) ", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "pxor %" VR0(r) ", %" VR2(r) "\n" \ + "pxor %" VR1(r) ", %" VR3(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define COPY(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 8: \ + __asm( \ + "movdqa %" VR0(r) ", %" VR4(r) "\n" \ + "movdqa %" VR1(r) ", %" VR5(r) "\n" \ + "movdqa %" VR2(r) ", %" VR6(r) "\n" \ + "movdqa %" VR3(r) ", %" VR7(r)); \ + break; \ + case 4: \ + __asm( \ + "movdqa %" VR0(r) ", %" VR2(r) "\n" \ + "movdqa %" VR1(r) ", %" VR3(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define LOAD(src, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ + "movdqa 0x10(%[SRC]), %%" VR1(r) "\n" \ + "movdqa 0x20(%[SRC]), %%" VR2(r) "\n" \ + "movdqa 0x30(%[SRC]), %%" VR3(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + case 2: \ + __asm( \ + "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ + "movdqa 0x10(%[SRC]), %%" VR1(r) "\n" \ + : : [SRC] "r" (src)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define STORE(dst, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + __asm( \ + "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ + "movdqa %%" VR1(r)", 0x10(%[DST])\n" \ + "movdqa %%" VR2(r)", 0x20(%[DST])\n" \ + "movdqa %%" VR3(r)", 0x30(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + case 2: \ + __asm( \ + "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ + "movdqa %%" VR1(r)", 0x10(%[DST])\n" \ + : : [DST] "r" (dst)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL2_SETUP() \ +{ \ + __asm( \ + "movd %[mask], %%xmm15\n" \ + "pshufd $0x0, %%xmm15, %%xmm15\n" \ + : : [mask] "r" (0x1d1d1d1d)); \ +} + +#define _MUL2_x2(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + __asm( \ + "pxor %xmm14, %xmm14\n" \ + "pxor %xmm13, %xmm13\n" \ + "pcmpgtb %" VR0(r)", %xmm14\n" \ + "pcmpgtb %" VR1(r)", %xmm13\n" \ + "pand %xmm15, %xmm14\n" \ + "pand %xmm15, %xmm13\n" \ + "paddb %" VR0(r)", %" VR0(r) "\n" \ + "paddb %" VR1(r)", %" VR1(r) "\n" \ + "pxor %xmm14, %" VR0(r) "\n" \ + "pxor %xmm13, %" VR1(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL2(r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + _MUL2_x2(R_01(r)); \ + _MUL2_x2(R_23(r)); \ + break; \ + case 2: \ + _MUL2_x2(r); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL4(r...) \ +{ \ + MUL2(r); \ + MUL2(r); \ +} + +#define _0f "xmm15" +#define _a_save "xmm14" +#define _b_save "xmm13" +#define _lt_mod_a "xmm12" +#define _lt_clmul_a "xmm11" +#define _lt_mod_b "xmm10" +#define _lt_clmul_b "xmm15" + +#define _MULx2(c, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 2: \ + __asm( \ + /* lts for upper part */ \ + "movd %[mask], %%" _0f "\n" \ + "pshufd $0x0, %%" _0f ", %%" _0f "\n" \ + "movdqa 0x00(%[lt]), %%" _lt_mod_a "\n" \ + "movdqa 0x10(%[lt]), %%" _lt_clmul_a "\n" \ + /* upper part */ \ + "movdqa %%" VR0(r) ", %%" _a_save "\n" \ + "movdqa %%" VR1(r) ", %%" _b_save "\n" \ + "psraw $0x4, %%" VR0(r) "\n" \ + "psraw $0x4, %%" VR1(r) "\n" \ + "pand %%" _0f ", %%" _a_save "\n" \ + "pand %%" _0f ", %%" _b_save "\n" \ + "pand %%" _0f ", %%" VR0(r) "\n" \ + "pand %%" _0f ", %%" VR1(r) "\n" \ + \ + "movdqa %%" _lt_mod_a ", %%" _lt_mod_b "\n" \ + "movdqa %%" _lt_clmul_a ", %%" _lt_clmul_b "\n" \ + \ + "pshufb %%" VR0(r) ",%%" _lt_mod_a "\n" \ + "pshufb %%" VR1(r) ",%%" _lt_mod_b "\n" \ + "pshufb %%" VR0(r) ",%%" _lt_clmul_a "\n" \ + "pshufb %%" VR1(r) ",%%" _lt_clmul_b "\n" \ + \ + "pxor %%" _lt_mod_a ",%%" _lt_clmul_a "\n" \ + "pxor %%" _lt_mod_b ",%%" _lt_clmul_b "\n" \ + "movdqa %%" _lt_clmul_a ",%%" VR0(r) "\n" \ + "movdqa %%" _lt_clmul_b ",%%" VR1(r) "\n" \ + /* lts for lower part */ \ + "movdqa 0x20(%[lt]), %%" _lt_mod_a "\n" \ + "movdqa 0x30(%[lt]), %%" _lt_clmul_a "\n" \ + "movdqa %%" _lt_mod_a ", %%" _lt_mod_b "\n" \ + "movdqa %%" _lt_clmul_a ", %%" _lt_clmul_b "\n" \ + /* lower part */ \ + "pshufb %%" _a_save ",%%" _lt_mod_a "\n" \ + "pshufb %%" _b_save ",%%" _lt_mod_b "\n" \ + "pshufb %%" _a_save ",%%" _lt_clmul_a "\n" \ + "pshufb %%" _b_save ",%%" _lt_clmul_b "\n" \ + \ + "pxor %%" _lt_mod_a ",%%" VR0(r) "\n" \ + "pxor %%" _lt_mod_b ",%%" VR1(r) "\n" \ + "pxor %%" _lt_clmul_a ",%%" VR0(r) "\n" \ + "pxor %%" _lt_clmul_b ",%%" VR1(r) "\n" \ + : : [mask] "r" (0x0f0f0f0f), \ + [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define MUL(c, r...) \ +{ \ + switch (REG_CNT(r)) { \ + case 4: \ + _MULx2(c, R_23(r)); \ + _MULx2(c, R_01(r)); \ + break; \ + case 2: \ + _MULx2(c, R_01(r)); \ + break; \ + default: \ + ASM_BUG(); \ + } \ +} + +#define raidz_math_begin() kfpu_begin() +#define raidz_math_end() kfpu_end() + +#define GEN_P_DEFINE() {} +#define GEN_P_STRIDE 4 +#define GEN_P_P 0, 1, 2, 3 + +#define GEN_PQ_DEFINE() {} +#define GEN_PQ_STRIDE 4 +#define GEN_PQ_D 0, 1, 2, 3 +#define GEN_PQ_P 4, 5, 6, 7 +#define GEN_PQ_Q 8, 9, 10, 11 + +#define GEN_PQR_DEFINE() {} +#define GEN_PQR_STRIDE 2 +#define GEN_PQR_D 0, 1 +#define GEN_PQR_P 2, 3 +#define GEN_PQR_Q 4, 5 +#define GEN_PQR_R 6, 7 + +#define REC_P_DEFINE() {} +#define REC_P_STRIDE 4 +#define REC_P_X 0, 1, 2, 3 + +#define REC_Q_DEFINE() {} +#define REC_Q_STRIDE 4 +#define REC_Q_X 0, 1, 2, 3 + +#define REC_R_DEFINE() {} +#define REC_R_STRIDE 4 +#define REC_R_X 0, 1, 2, 3 + +#define REC_PQ_DEFINE() {} +#define REC_PQ_STRIDE 2 +#define REC_PQ_X 0, 1 +#define REC_PQ_Y 2, 3 +#define REC_PQ_D 4, 5 + +#define REC_PR_DEFINE() {} +#define REC_PR_STRIDE 2 +#define REC_PR_X 0, 1 +#define REC_PR_Y 2, 3 +#define REC_PR_D 4, 5 + +#define REC_QR_DEFINE() {} +#define REC_QR_STRIDE 2 +#define REC_QR_X 0, 1 +#define REC_QR_Y 2, 3 +#define REC_QR_D 4, 5 + +#define REC_PQR_DEFINE() {} +#define REC_PQR_STRIDE 2 +#define REC_PQR_X 0, 1 +#define REC_PQR_Y 2, 3 +#define REC_PQR_Z 4, 5 +#define REC_PQR_D 6, 7 +#define REC_PQR_XS 6, 7 +#define REC_PQR_YS 8, 9 + + +#include +#include "vdev_raidz_math_impl.h" + +DEFINE_GEN_METHODS(ssse3); +DEFINE_REC_METHODS(ssse3); + +static boolean_t +raidz_will_ssse3_work(void) +{ + return (zfs_sse_available() && zfs_sse2_available() && + zfs_ssse3_available()); +} + +const raidz_impl_ops_t vdev_raidz_ssse3_impl = { + .init = NULL, + .fini = NULL, + .gen = RAIDZ_GEN_METHODS(ssse3), + .rec = RAIDZ_REC_METHODS(ssse3), + .is_supported = &raidz_will_ssse3_work, + .name = "ssse3" +}; + +#endif /* defined(__x86_64) && defined(HAVE_SSSE3) */ + + +#if defined(__x86_64) && (defined(HAVE_SSSE3) || defined(HAVE_AVX2)) + +const uint8_t +__attribute__((aligned(256))) gf_clmul_mod_lt[4*256][16] = { + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e, + 0x10, 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x03, 0x06, 0x05, 0x0c, 0x0f, 0x0a, 0x09, + 0x18, 0x1b, 0x1e, 0x1d, 0x14, 0x17, 0x12, 0x11 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x04, 0x08, 0x0c, 0x10, 0x14, 0x18, 0x1c, + 0x20, 0x24, 0x28, 0x2c, 0x30, 0x34, 0x38, 0x3c }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x05, 0x0a, 0x0f, 0x14, 0x11, 0x1e, 0x1b, + 0x28, 0x2d, 0x22, 0x27, 0x3c, 0x39, 0x36, 0x33 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x06, 0x0c, 0x0a, 0x18, 0x1e, 0x14, 0x12, + 0x30, 0x36, 0x3c, 0x3a, 0x28, 0x2e, 0x24, 0x22 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x07, 0x0e, 0x09, 0x1c, 0x1b, 0x12, 0x15, + 0x38, 0x3f, 0x36, 0x31, 0x24, 0x23, 0x2a, 0x2d }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x08, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38, + 0x40, 0x48, 0x50, 0x58, 0x60, 0x68, 0x70, 0x78 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x09, 0x12, 0x1b, 0x24, 0x2d, 0x36, 0x3f, + 0x48, 0x41, 0x5a, 0x53, 0x6c, 0x65, 0x7e, 0x77 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x0a, 0x14, 0x1e, 0x28, 0x22, 0x3c, 0x36, + 0x50, 0x5a, 0x44, 0x4e, 0x78, 0x72, 0x6c, 0x66 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x0b, 0x16, 0x1d, 0x2c, 0x27, 0x3a, 0x31, + 0x58, 0x53, 0x4e, 0x45, 0x74, 0x7f, 0x62, 0x69 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x0c, 0x18, 0x14, 0x30, 0x3c, 0x28, 0x24, + 0x60, 0x6c, 0x78, 0x74, 0x50, 0x5c, 0x48, 0x44 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x0d, 0x1a, 0x17, 0x34, 0x39, 0x2e, 0x23, + 0x68, 0x65, 0x72, 0x7f, 0x5c, 0x51, 0x46, 0x4b }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x0e, 0x1c, 0x12, 0x38, 0x36, 0x24, 0x2a, + 0x70, 0x7e, 0x6c, 0x62, 0x48, 0x46, 0x54, 0x5a }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x0f, 0x1e, 0x11, 0x3c, 0x33, 0x22, 0x2d, + 0x78, 0x77, 0x66, 0x69, 0x44, 0x4b, 0x5a, 0x55 }, + { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, + 0xe8, 0xf5, 0xd2, 0xcf, 0x9c, 0x81, 0xa6, 0xbb }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, + 0xe8, 0xf5, 0xd2, 0xcf, 0x9c, 0x81, 0xa6, 0xbb }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, + 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff }, + { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, + 0xf5, 0xe8, 0xcf, 0xd2, 0x81, 0x9c, 0xbb, 0xa6 }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x12, 0x24, 0x36, 0x48, 0x5a, 0x6c, 0x7e, + 0x90, 0x82, 0xb4, 0xa6, 0xd8, 0xca, 0xfc, 0xee }, + { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, + 0xf5, 0xe8, 0xcf, 0xd2, 0x81, 0x9c, 0xbb, 0xa6 }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x13, 0x26, 0x35, 0x4c, 0x5f, 0x6a, 0x79, + 0x98, 0x8b, 0xbe, 0xad, 0xd4, 0xc7, 0xf2, 0xe1 }, + { 0x00, 0x1d, 0x3a, 0x27, 0x69, 0x74, 0x53, 0x4e, + 0xd2, 0xcf, 0xe8, 0xf5, 0xbb, 0xa6, 0x81, 0x9c }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x14, 0x28, 0x3c, 0x50, 0x44, 0x78, 0x6c, + 0xa0, 0xb4, 0x88, 0x9c, 0xf0, 0xe4, 0xd8, 0xcc }, + { 0x00, 0x1d, 0x3a, 0x27, 0x69, 0x74, 0x53, 0x4e, + 0xd2, 0xcf, 0xe8, 0xf5, 0xbb, 0xa6, 0x81, 0x9c }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x15, 0x2a, 0x3f, 0x54, 0x41, 0x7e, 0x6b, + 0xa8, 0xbd, 0x82, 0x97, 0xfc, 0xe9, 0xd6, 0xc3 }, + { 0x00, 0x1d, 0x3a, 0x27, 0x69, 0x74, 0x53, 0x4e, + 0xcf, 0xd2, 0xf5, 0xe8, 0xa6, 0xbb, 0x9c, 0x81 }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x16, 0x2c, 0x3a, 0x58, 0x4e, 0x74, 0x62, + 0xb0, 0xa6, 0x9c, 0x8a, 0xe8, 0xfe, 0xc4, 0xd2 }, + { 0x00, 0x1d, 0x3a, 0x27, 0x69, 0x74, 0x53, 0x4e, + 0xcf, 0xd2, 0xf5, 0xe8, 0xa6, 0xbb, 0x9c, 0x81 }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x17, 0x2e, 0x39, 0x5c, 0x4b, 0x72, 0x65, + 0xb8, 0xaf, 0x96, 0x81, 0xe4, 0xf3, 0xca, 0xdd }, + { 0x00, 0x1d, 0x27, 0x3a, 0x4e, 0x53, 0x69, 0x74, + 0x9c, 0x81, 0xbb, 0xa6, 0xd2, 0xcf, 0xf5, 0xe8 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x18, 0x30, 0x28, 0x60, 0x78, 0x50, 0x48, + 0xc0, 0xd8, 0xf0, 0xe8, 0xa0, 0xb8, 0x90, 0x88 }, + { 0x00, 0x1d, 0x27, 0x3a, 0x4e, 0x53, 0x69, 0x74, + 0x9c, 0x81, 0xbb, 0xa6, 0xd2, 0xcf, 0xf5, 0xe8 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x19, 0x32, 0x2b, 0x64, 0x7d, 0x56, 0x4f, + 0xc8, 0xd1, 0xfa, 0xe3, 0xac, 0xb5, 0x9e, 0x87 }, + { 0x00, 0x1d, 0x27, 0x3a, 0x4e, 0x53, 0x69, 0x74, + 0x81, 0x9c, 0xa6, 0xbb, 0xcf, 0xd2, 0xe8, 0xf5 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x1a, 0x34, 0x2e, 0x68, 0x72, 0x5c, 0x46, + 0xd0, 0xca, 0xe4, 0xfe, 0xb8, 0xa2, 0x8c, 0x96 }, + { 0x00, 0x1d, 0x27, 0x3a, 0x4e, 0x53, 0x69, 0x74, + 0x81, 0x9c, 0xa6, 0xbb, 0xcf, 0xd2, 0xe8, 0xf5 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x1b, 0x36, 0x2d, 0x6c, 0x77, 0x5a, 0x41, + 0xd8, 0xc3, 0xee, 0xf5, 0xb4, 0xaf, 0x82, 0x99 }, + { 0x00, 0x1d, 0x27, 0x3a, 0x53, 0x4e, 0x74, 0x69, + 0xa6, 0xbb, 0x81, 0x9c, 0xf5, 0xe8, 0xd2, 0xcf }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x1c, 0x38, 0x24, 0x70, 0x6c, 0x48, 0x54, + 0xe0, 0xfc, 0xd8, 0xc4, 0x90, 0x8c, 0xa8, 0xb4 }, + { 0x00, 0x1d, 0x27, 0x3a, 0x53, 0x4e, 0x74, 0x69, + 0xa6, 0xbb, 0x81, 0x9c, 0xf5, 0xe8, 0xd2, 0xcf }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, + 0xe8, 0xf5, 0xd2, 0xcf, 0x9c, 0x81, 0xa6, 0xbb }, + { 0x00, 0x1d, 0x27, 0x3a, 0x53, 0x4e, 0x74, 0x69, + 0xbb, 0xa6, 0x9c, 0x81, 0xe8, 0xf5, 0xcf, 0xd2 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x1e, 0x3c, 0x22, 0x78, 0x66, 0x44, 0x5a, + 0xf0, 0xee, 0xcc, 0xd2, 0x88, 0x96, 0xb4, 0xaa }, + { 0x00, 0x1d, 0x27, 0x3a, 0x53, 0x4e, 0x74, 0x69, + 0xbb, 0xa6, 0x9c, 0x81, 0xe8, 0xf5, 0xcf, 0xd2 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x1f, 0x3e, 0x21, 0x7c, 0x63, 0x42, 0x5d, + 0xf8, 0xe7, 0xc6, 0xd9, 0x84, 0x9b, 0xba, 0xa5 }, + { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, + 0xcd, 0xf7, 0xb9, 0x83, 0x25, 0x1f, 0x51, 0x6b }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, + 0xcd, 0xf7, 0xb9, 0x83, 0x25, 0x1f, 0x51, 0x6b }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x21, 0x42, 0x63, 0x84, 0xa5, 0xc6, 0xe7, + 0x08, 0x29, 0x4a, 0x6b, 0x8c, 0xad, 0xce, 0xef }, + { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, + 0xd0, 0xea, 0xa4, 0x9e, 0x38, 0x02, 0x4c, 0x76 }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x22, 0x44, 0x66, 0x88, 0xaa, 0xcc, 0xee, + 0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe }, + { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, + 0xd0, 0xea, 0xa4, 0x9e, 0x38, 0x02, 0x4c, 0x76 }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x23, 0x46, 0x65, 0x8c, 0xaf, 0xca, 0xe9, + 0x18, 0x3b, 0x5e, 0x7d, 0x94, 0xb7, 0xd2, 0xf1 }, + { 0x00, 0x3a, 0x74, 0x4e, 0xf5, 0xcf, 0x81, 0xbb, + 0xf7, 0xcd, 0x83, 0xb9, 0x02, 0x38, 0x76, 0x4c }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x24, 0x48, 0x6c, 0x90, 0xb4, 0xd8, 0xfc, + 0x20, 0x04, 0x68, 0x4c, 0xb0, 0x94, 0xf8, 0xdc }, + { 0x00, 0x3a, 0x74, 0x4e, 0xf5, 0xcf, 0x81, 0xbb, + 0xf7, 0xcd, 0x83, 0xb9, 0x02, 0x38, 0x76, 0x4c }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x25, 0x4a, 0x6f, 0x94, 0xb1, 0xde, 0xfb, + 0x28, 0x0d, 0x62, 0x47, 0xbc, 0x99, 0xf6, 0xd3 }, + { 0x00, 0x3a, 0x74, 0x4e, 0xf5, 0xcf, 0x81, 0xbb, + 0xea, 0xd0, 0x9e, 0xa4, 0x1f, 0x25, 0x6b, 0x51 }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x26, 0x4c, 0x6a, 0x98, 0xbe, 0xd4, 0xf2, + 0x30, 0x16, 0x7c, 0x5a, 0xa8, 0x8e, 0xe4, 0xc2 }, + { 0x00, 0x3a, 0x74, 0x4e, 0xf5, 0xcf, 0x81, 0xbb, + 0xea, 0xd0, 0x9e, 0xa4, 0x1f, 0x25, 0x6b, 0x51 }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, + 0x38, 0x1f, 0x76, 0x51, 0xa4, 0x83, 0xea, 0xcd }, + { 0x00, 0x3a, 0x69, 0x53, 0xd2, 0xe8, 0xbb, 0x81, + 0xb9, 0x83, 0xd0, 0xea, 0x6b, 0x51, 0x02, 0x38 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x28, 0x50, 0x78, 0xa0, 0x88, 0xf0, 0xd8, + 0x40, 0x68, 0x10, 0x38, 0xe0, 0xc8, 0xb0, 0x98 }, + { 0x00, 0x3a, 0x69, 0x53, 0xd2, 0xe8, 0xbb, 0x81, + 0xb9, 0x83, 0xd0, 0xea, 0x6b, 0x51, 0x02, 0x38 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x29, 0x52, 0x7b, 0xa4, 0x8d, 0xf6, 0xdf, + 0x48, 0x61, 0x1a, 0x33, 0xec, 0xc5, 0xbe, 0x97 }, + { 0x00, 0x3a, 0x69, 0x53, 0xd2, 0xe8, 0xbb, 0x81, + 0xa4, 0x9e, 0xcd, 0xf7, 0x76, 0x4c, 0x1f, 0x25 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x2a, 0x54, 0x7e, 0xa8, 0x82, 0xfc, 0xd6, + 0x50, 0x7a, 0x04, 0x2e, 0xf8, 0xd2, 0xac, 0x86 }, + { 0x00, 0x3a, 0x69, 0x53, 0xd2, 0xe8, 0xbb, 0x81, + 0xa4, 0x9e, 0xcd, 0xf7, 0x76, 0x4c, 0x1f, 0x25 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x2b, 0x56, 0x7d, 0xac, 0x87, 0xfa, 0xd1, + 0x58, 0x73, 0x0e, 0x25, 0xf4, 0xdf, 0xa2, 0x89 }, + { 0x00, 0x3a, 0x69, 0x53, 0xcf, 0xf5, 0xa6, 0x9c, + 0x83, 0xb9, 0xea, 0xd0, 0x4c, 0x76, 0x25, 0x1f }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x2c, 0x58, 0x74, 0xb0, 0x9c, 0xe8, 0xc4, + 0x60, 0x4c, 0x38, 0x14, 0xd0, 0xfc, 0x88, 0xa4 }, + { 0x00, 0x3a, 0x69, 0x53, 0xcf, 0xf5, 0xa6, 0x9c, + 0x83, 0xb9, 0xea, 0xd0, 0x4c, 0x76, 0x25, 0x1f }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x2d, 0x5a, 0x77, 0xb4, 0x99, 0xee, 0xc3, + 0x68, 0x45, 0x32, 0x1f, 0xdc, 0xf1, 0x86, 0xab }, + { 0x00, 0x3a, 0x69, 0x53, 0xcf, 0xf5, 0xa6, 0x9c, + 0x9e, 0xa4, 0xf7, 0xcd, 0x51, 0x6b, 0x38, 0x02 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x2e, 0x5c, 0x72, 0xb8, 0x96, 0xe4, 0xca, + 0x70, 0x5e, 0x2c, 0x02, 0xc8, 0xe6, 0x94, 0xba }, + { 0x00, 0x3a, 0x69, 0x53, 0xcf, 0xf5, 0xa6, 0x9c, + 0x9e, 0xa4, 0xf7, 0xcd, 0x51, 0x6b, 0x38, 0x02 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x2f, 0x5e, 0x71, 0xbc, 0x93, 0xe2, 0xcd, + 0x78, 0x57, 0x26, 0x09, 0xc4, 0xeb, 0x9a, 0xb5 }, + { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, + 0x25, 0x02, 0x6b, 0x4c, 0xb9, 0x9e, 0xf7, 0xd0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, + 0x25, 0x02, 0x6b, 0x4c, 0xb9, 0x9e, 0xf7, 0xd0 }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x31, 0x62, 0x53, 0xc4, 0xf5, 0xa6, 0x97, + 0x88, 0xb9, 0xea, 0xdb, 0x4c, 0x7d, 0x2e, 0x1f }, + { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, + 0x38, 0x1f, 0x76, 0x51, 0xa4, 0x83, 0xea, 0xcd }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x32, 0x64, 0x56, 0xc8, 0xfa, 0xac, 0x9e, + 0x90, 0xa2, 0xf4, 0xc6, 0x58, 0x6a, 0x3c, 0x0e }, + { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, + 0x38, 0x1f, 0x76, 0x51, 0xa4, 0x83, 0xea, 0xcd }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x33, 0x66, 0x55, 0xcc, 0xff, 0xaa, 0x99, + 0x98, 0xab, 0xfe, 0xcd, 0x54, 0x67, 0x32, 0x01 }, + { 0x00, 0x27, 0x4e, 0x69, 0x81, 0xa6, 0xcf, 0xe8, + 0x1f, 0x38, 0x51, 0x76, 0x9e, 0xb9, 0xd0, 0xf7 }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x34, 0x68, 0x5c, 0xd0, 0xe4, 0xb8, 0x8c, + 0xa0, 0x94, 0xc8, 0xfc, 0x70, 0x44, 0x18, 0x2c }, + { 0x00, 0x27, 0x4e, 0x69, 0x81, 0xa6, 0xcf, 0xe8, + 0x1f, 0x38, 0x51, 0x76, 0x9e, 0xb9, 0xd0, 0xf7 }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x35, 0x6a, 0x5f, 0xd4, 0xe1, 0xbe, 0x8b, + 0xa8, 0x9d, 0xc2, 0xf7, 0x7c, 0x49, 0x16, 0x23 }, + { 0x00, 0x27, 0x4e, 0x69, 0x81, 0xa6, 0xcf, 0xe8, + 0x02, 0x25, 0x4c, 0x6b, 0x83, 0xa4, 0xcd, 0xea }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x36, 0x6c, 0x5a, 0xd8, 0xee, 0xb4, 0x82, + 0xb0, 0x86, 0xdc, 0xea, 0x68, 0x5e, 0x04, 0x32 }, + { 0x00, 0x27, 0x4e, 0x69, 0x81, 0xa6, 0xcf, 0xe8, + 0x02, 0x25, 0x4c, 0x6b, 0x83, 0xa4, 0xcd, 0xea }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x37, 0x6e, 0x59, 0xdc, 0xeb, 0xb2, 0x85, + 0xb8, 0x8f, 0xd6, 0xe1, 0x64, 0x53, 0x0a, 0x3d }, + { 0x00, 0x27, 0x53, 0x74, 0xa6, 0x81, 0xf5, 0xd2, + 0x51, 0x76, 0x02, 0x25, 0xf7, 0xd0, 0xa4, 0x83 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x38, 0x70, 0x48, 0xe0, 0xd8, 0x90, 0xa8, + 0xc0, 0xf8, 0xb0, 0x88, 0x20, 0x18, 0x50, 0x68 }, + { 0x00, 0x27, 0x53, 0x74, 0xa6, 0x81, 0xf5, 0xd2, + 0x51, 0x76, 0x02, 0x25, 0xf7, 0xd0, 0xa4, 0x83 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x39, 0x72, 0x4b, 0xe4, 0xdd, 0x96, 0xaf, + 0xc8, 0xf1, 0xba, 0x83, 0x2c, 0x15, 0x5e, 0x67 }, + { 0x00, 0x27, 0x53, 0x74, 0xa6, 0x81, 0xf5, 0xd2, + 0x4c, 0x6b, 0x1f, 0x38, 0xea, 0xcd, 0xb9, 0x9e }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, + 0xd0, 0xea, 0xa4, 0x9e, 0x38, 0x02, 0x4c, 0x76 }, + { 0x00, 0x27, 0x53, 0x74, 0xa6, 0x81, 0xf5, 0xd2, + 0x4c, 0x6b, 0x1f, 0x38, 0xea, 0xcd, 0xb9, 0x9e }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x3b, 0x76, 0x4d, 0xec, 0xd7, 0x9a, 0xa1, + 0xd8, 0xe3, 0xae, 0x95, 0x34, 0x0f, 0x42, 0x79 }, + { 0x00, 0x27, 0x53, 0x74, 0xbb, 0x9c, 0xe8, 0xcf, + 0x6b, 0x4c, 0x38, 0x1f, 0xd0, 0xf7, 0x83, 0xa4 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x3c, 0x78, 0x44, 0xf0, 0xcc, 0x88, 0xb4, + 0xe0, 0xdc, 0x98, 0xa4, 0x10, 0x2c, 0x68, 0x54 }, + { 0x00, 0x27, 0x53, 0x74, 0xbb, 0x9c, 0xe8, 0xcf, + 0x6b, 0x4c, 0x38, 0x1f, 0xd0, 0xf7, 0x83, 0xa4 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x3d, 0x7a, 0x47, 0xf4, 0xc9, 0x8e, 0xb3, + 0xe8, 0xd5, 0x92, 0xaf, 0x1c, 0x21, 0x66, 0x5b }, + { 0x00, 0x27, 0x53, 0x74, 0xbb, 0x9c, 0xe8, 0xcf, + 0x76, 0x51, 0x25, 0x02, 0xcd, 0xea, 0x9e, 0xb9 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x3e, 0x7c, 0x42, 0xf8, 0xc6, 0x84, 0xba, + 0xf0, 0xce, 0x8c, 0xb2, 0x08, 0x36, 0x74, 0x4a }, + { 0x00, 0x27, 0x53, 0x74, 0xbb, 0x9c, 0xe8, 0xcf, + 0x76, 0x51, 0x25, 0x02, 0xcd, 0xea, 0x9e, 0xb9 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x00, 0x3f, 0x7e, 0x41, 0xfc, 0xc3, 0x82, 0xbd, + 0xf8, 0xc7, 0x86, 0xb9, 0x04, 0x3b, 0x7a, 0x45 }, + { 0x00, 0x74, 0xe8, 0x9c, 0xcd, 0xb9, 0x25, 0x51, + 0x87, 0xf3, 0x6f, 0x1b, 0x4a, 0x3e, 0xa2, 0xd6 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x74, 0xe8, 0x9c, 0xcd, 0xb9, 0x25, 0x51, + 0x87, 0xf3, 0x6f, 0x1b, 0x4a, 0x3e, 0xa2, 0xd6 }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x41, 0x82, 0xc3, 0x04, 0x45, 0x86, 0xc7, + 0x08, 0x49, 0x8a, 0xcb, 0x0c, 0x4d, 0x8e, 0xcf }, + { 0x00, 0x74, 0xe8, 0x9c, 0xcd, 0xb9, 0x25, 0x51, + 0x9a, 0xee, 0x72, 0x06, 0x57, 0x23, 0xbf, 0xcb }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x42, 0x84, 0xc6, 0x08, 0x4a, 0x8c, 0xce, + 0x10, 0x52, 0x94, 0xd6, 0x18, 0x5a, 0x9c, 0xde }, + { 0x00, 0x74, 0xe8, 0x9c, 0xcd, 0xb9, 0x25, 0x51, + 0x9a, 0xee, 0x72, 0x06, 0x57, 0x23, 0xbf, 0xcb }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x43, 0x86, 0xc5, 0x0c, 0x4f, 0x8a, 0xc9, + 0x18, 0x5b, 0x9e, 0xdd, 0x14, 0x57, 0x92, 0xd1 }, + { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, + 0xbd, 0xc9, 0x55, 0x21, 0x6d, 0x19, 0x85, 0xf1 }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x44, 0x88, 0xcc, 0x10, 0x54, 0x98, 0xdc, + 0x20, 0x64, 0xa8, 0xec, 0x30, 0x74, 0xb8, 0xfc }, + { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, + 0xbd, 0xc9, 0x55, 0x21, 0x6d, 0x19, 0x85, 0xf1 }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x45, 0x8a, 0xcf, 0x14, 0x51, 0x9e, 0xdb, + 0x28, 0x6d, 0xa2, 0xe7, 0x3c, 0x79, 0xb6, 0xf3 }, + { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, + 0xa0, 0xd4, 0x48, 0x3c, 0x70, 0x04, 0x98, 0xec }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x46, 0x8c, 0xca, 0x18, 0x5e, 0x94, 0xd2, + 0x30, 0x76, 0xbc, 0xfa, 0x28, 0x6e, 0xa4, 0xe2 }, + { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, + 0xa0, 0xd4, 0x48, 0x3c, 0x70, 0x04, 0x98, 0xec }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x47, 0x8e, 0xc9, 0x1c, 0x5b, 0x92, 0xd5, + 0x38, 0x7f, 0xb6, 0xf1, 0x24, 0x63, 0xaa, 0xed }, + { 0x00, 0x74, 0xf5, 0x81, 0xf7, 0x83, 0x02, 0x76, + 0xf3, 0x87, 0x06, 0x72, 0x04, 0x70, 0xf1, 0x85 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x48, 0x90, 0xd8, 0x20, 0x68, 0xb0, 0xf8, + 0x40, 0x08, 0xd0, 0x98, 0x60, 0x28, 0xf0, 0xb8 }, + { 0x00, 0x74, 0xf5, 0x81, 0xf7, 0x83, 0x02, 0x76, + 0xf3, 0x87, 0x06, 0x72, 0x04, 0x70, 0xf1, 0x85 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x49, 0x92, 0xdb, 0x24, 0x6d, 0xb6, 0xff, + 0x48, 0x01, 0xda, 0x93, 0x6c, 0x25, 0xfe, 0xb7 }, + { 0x00, 0x74, 0xf5, 0x81, 0xf7, 0x83, 0x02, 0x76, + 0xee, 0x9a, 0x1b, 0x6f, 0x19, 0x6d, 0xec, 0x98 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x4a, 0x94, 0xde, 0x28, 0x62, 0xbc, 0xf6, + 0x50, 0x1a, 0xc4, 0x8e, 0x78, 0x32, 0xec, 0xa6 }, + { 0x00, 0x74, 0xf5, 0x81, 0xf7, 0x83, 0x02, 0x76, + 0xee, 0x9a, 0x1b, 0x6f, 0x19, 0x6d, 0xec, 0x98 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x4b, 0x96, 0xdd, 0x2c, 0x67, 0xba, 0xf1, + 0x58, 0x13, 0xce, 0x85, 0x74, 0x3f, 0xe2, 0xa9 }, + { 0x00, 0x74, 0xf5, 0x81, 0xea, 0x9e, 0x1f, 0x6b, + 0xc9, 0xbd, 0x3c, 0x48, 0x23, 0x57, 0xd6, 0xa2 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x4c, 0x98, 0xd4, 0x30, 0x7c, 0xa8, 0xe4, + 0x60, 0x2c, 0xf8, 0xb4, 0x50, 0x1c, 0xc8, 0x84 }, + { 0x00, 0x74, 0xf5, 0x81, 0xea, 0x9e, 0x1f, 0x6b, + 0xc9, 0xbd, 0x3c, 0x48, 0x23, 0x57, 0xd6, 0xa2 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x4d, 0x9a, 0xd7, 0x34, 0x79, 0xae, 0xe3, + 0x68, 0x25, 0xf2, 0xbf, 0x5c, 0x11, 0xc6, 0x8b }, + { 0x00, 0x74, 0xf5, 0x81, 0xea, 0x9e, 0x1f, 0x6b, + 0xd4, 0xa0, 0x21, 0x55, 0x3e, 0x4a, 0xcb, 0xbf }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, + 0x70, 0x3e, 0xec, 0xa2, 0x48, 0x06, 0xd4, 0x9a }, + { 0x00, 0x74, 0xf5, 0x81, 0xea, 0x9e, 0x1f, 0x6b, + 0xd4, 0xa0, 0x21, 0x55, 0x3e, 0x4a, 0xcb, 0xbf }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x4f, 0x9e, 0xd1, 0x3c, 0x73, 0xa2, 0xed, + 0x78, 0x37, 0xe6, 0xa9, 0x44, 0x0b, 0xda, 0x95 }, + { 0x00, 0x69, 0xd2, 0xbb, 0xb9, 0xd0, 0x6b, 0x02, + 0x6f, 0x06, 0xbd, 0xd4, 0xd6, 0xbf, 0x04, 0x6d }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x69, 0xd2, 0xbb, 0xb9, 0xd0, 0x6b, 0x02, + 0x6f, 0x06, 0xbd, 0xd4, 0xd6, 0xbf, 0x04, 0x6d }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x51, 0xa2, 0xf3, 0x44, 0x15, 0xe6, 0xb7, + 0x88, 0xd9, 0x2a, 0x7b, 0xcc, 0x9d, 0x6e, 0x3f }, + { 0x00, 0x69, 0xd2, 0xbb, 0xb9, 0xd0, 0x6b, 0x02, + 0x72, 0x1b, 0xa0, 0xc9, 0xcb, 0xa2, 0x19, 0x70 }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x52, 0xa4, 0xf6, 0x48, 0x1a, 0xec, 0xbe, + 0x90, 0xc2, 0x34, 0x66, 0xd8, 0x8a, 0x7c, 0x2e }, + { 0x00, 0x69, 0xd2, 0xbb, 0xb9, 0xd0, 0x6b, 0x02, + 0x72, 0x1b, 0xa0, 0xc9, 0xcb, 0xa2, 0x19, 0x70 }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, + 0x98, 0xcb, 0x3e, 0x6d, 0xd4, 0x87, 0x72, 0x21 }, + { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, + 0x55, 0x3c, 0x87, 0xee, 0xf1, 0x98, 0x23, 0x4a }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x54, 0xa8, 0xfc, 0x50, 0x04, 0xf8, 0xac, + 0xa0, 0xf4, 0x08, 0x5c, 0xf0, 0xa4, 0x58, 0x0c }, + { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, + 0x55, 0x3c, 0x87, 0xee, 0xf1, 0x98, 0x23, 0x4a }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x55, 0xaa, 0xff, 0x54, 0x01, 0xfe, 0xab, + 0xa8, 0xfd, 0x02, 0x57, 0xfc, 0xa9, 0x56, 0x03 }, + { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, + 0x48, 0x21, 0x9a, 0xf3, 0xec, 0x85, 0x3e, 0x57 }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x56, 0xac, 0xfa, 0x58, 0x0e, 0xf4, 0xa2, + 0xb0, 0xe6, 0x1c, 0x4a, 0xe8, 0xbe, 0x44, 0x12 }, + { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, + 0x48, 0x21, 0x9a, 0xf3, 0xec, 0x85, 0x3e, 0x57 }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x57, 0xae, 0xf9, 0x5c, 0x0b, 0xf2, 0xa5, + 0xb8, 0xef, 0x16, 0x41, 0xe4, 0xb3, 0x4a, 0x1d }, + { 0x00, 0x69, 0xcf, 0xa6, 0x83, 0xea, 0x4c, 0x25, + 0x1b, 0x72, 0xd4, 0xbd, 0x98, 0xf1, 0x57, 0x3e }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x58, 0xb0, 0xe8, 0x60, 0x38, 0xd0, 0x88, + 0xc0, 0x98, 0x70, 0x28, 0xa0, 0xf8, 0x10, 0x48 }, + { 0x00, 0x69, 0xcf, 0xa6, 0x83, 0xea, 0x4c, 0x25, + 0x1b, 0x72, 0xd4, 0xbd, 0x98, 0xf1, 0x57, 0x3e }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x59, 0xb2, 0xeb, 0x64, 0x3d, 0xd6, 0x8f, + 0xc8, 0x91, 0x7a, 0x23, 0xac, 0xf5, 0x1e, 0x47 }, + { 0x00, 0x69, 0xcf, 0xa6, 0x83, 0xea, 0x4c, 0x25, + 0x06, 0x6f, 0xc9, 0xa0, 0x85, 0xec, 0x4a, 0x23 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x5a, 0xb4, 0xee, 0x68, 0x32, 0xdc, 0x86, + 0xd0, 0x8a, 0x64, 0x3e, 0xb8, 0xe2, 0x0c, 0x56 }, + { 0x00, 0x69, 0xcf, 0xa6, 0x83, 0xea, 0x4c, 0x25, + 0x06, 0x6f, 0xc9, 0xa0, 0x85, 0xec, 0x4a, 0x23 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x5b, 0xb6, 0xed, 0x6c, 0x37, 0xda, 0x81, + 0xd8, 0x83, 0x6e, 0x35, 0xb4, 0xef, 0x02, 0x59 }, + { 0x00, 0x69, 0xcf, 0xa6, 0x9e, 0xf7, 0x51, 0x38, + 0x21, 0x48, 0xee, 0x87, 0xbf, 0xd6, 0x70, 0x19 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x5c, 0xb8, 0xe4, 0x70, 0x2c, 0xc8, 0x94, + 0xe0, 0xbc, 0x58, 0x04, 0x90, 0xcc, 0x28, 0x74 }, + { 0x00, 0x69, 0xcf, 0xa6, 0x9e, 0xf7, 0x51, 0x38, + 0x21, 0x48, 0xee, 0x87, 0xbf, 0xd6, 0x70, 0x19 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x5d, 0xba, 0xe7, 0x74, 0x29, 0xce, 0x93, + 0xe8, 0xb5, 0x52, 0x0f, 0x9c, 0xc1, 0x26, 0x7b }, + { 0x00, 0x69, 0xcf, 0xa6, 0x9e, 0xf7, 0x51, 0x38, + 0x3c, 0x55, 0xf3, 0x9a, 0xa2, 0xcb, 0x6d, 0x04 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x5e, 0xbc, 0xe2, 0x78, 0x26, 0xc4, 0x9a, + 0xf0, 0xae, 0x4c, 0x12, 0x88, 0xd6, 0x34, 0x6a }, + { 0x00, 0x69, 0xcf, 0xa6, 0x9e, 0xf7, 0x51, 0x38, + 0x3c, 0x55, 0xf3, 0x9a, 0xa2, 0xcb, 0x6d, 0x04 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, + { 0x00, 0x5f, 0xbe, 0xe1, 0x7c, 0x23, 0xc2, 0x9d, + 0xf8, 0xa7, 0x46, 0x19, 0x84, 0xdb, 0x3a, 0x65 }, + { 0x00, 0x4e, 0x9c, 0xd2, 0x25, 0x6b, 0xb9, 0xf7, + 0x4a, 0x04, 0xd6, 0x98, 0x6f, 0x21, 0xf3, 0xbd }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x4e, 0x9c, 0xd2, 0x25, 0x6b, 0xb9, 0xf7, + 0x4a, 0x04, 0xd6, 0x98, 0x6f, 0x21, 0xf3, 0xbd }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x61, 0xc2, 0xa3, 0x84, 0xe5, 0x46, 0x27, + 0x08, 0x69, 0xca, 0xab, 0x8c, 0xed, 0x4e, 0x2f }, + { 0x00, 0x4e, 0x9c, 0xd2, 0x25, 0x6b, 0xb9, 0xf7, + 0x57, 0x19, 0xcb, 0x85, 0x72, 0x3c, 0xee, 0xa0 }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x62, 0xc4, 0xa6, 0x88, 0xea, 0x4c, 0x2e, + 0x10, 0x72, 0xd4, 0xb6, 0x98, 0xfa, 0x5c, 0x3e }, + { 0x00, 0x4e, 0x9c, 0xd2, 0x25, 0x6b, 0xb9, 0xf7, + 0x57, 0x19, 0xcb, 0x85, 0x72, 0x3c, 0xee, 0xa0 }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x63, 0xc6, 0xa5, 0x8c, 0xef, 0x4a, 0x29, + 0x18, 0x7b, 0xde, 0xbd, 0x94, 0xf7, 0x52, 0x31 }, + { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, + 0x70, 0x3e, 0xec, 0xa2, 0x48, 0x06, 0xd4, 0x9a }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x64, 0xc8, 0xac, 0x90, 0xf4, 0x58, 0x3c, + 0x20, 0x44, 0xe8, 0x8c, 0xb0, 0xd4, 0x78, 0x1c }, + { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, + 0x70, 0x3e, 0xec, 0xa2, 0x48, 0x06, 0xd4, 0x9a }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x65, 0xca, 0xaf, 0x94, 0xf1, 0x5e, 0x3b, + 0x28, 0x4d, 0xe2, 0x87, 0xbc, 0xd9, 0x76, 0x13 }, + { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, + 0x6d, 0x23, 0xf1, 0xbf, 0x55, 0x1b, 0xc9, 0x87 }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x66, 0xcc, 0xaa, 0x98, 0xfe, 0x54, 0x32, + 0x30, 0x56, 0xfc, 0x9a, 0xa8, 0xce, 0x64, 0x02 }, + { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, + 0x6d, 0x23, 0xf1, 0xbf, 0x55, 0x1b, 0xc9, 0x87 }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x67, 0xce, 0xa9, 0x9c, 0xfb, 0x52, 0x35, + 0x38, 0x5f, 0xf6, 0x91, 0xa4, 0xc3, 0x6a, 0x0d }, + { 0x00, 0x4e, 0x81, 0xcf, 0x1f, 0x51, 0x9e, 0xd0, + 0x3e, 0x70, 0xbf, 0xf1, 0x21, 0x6f, 0xa0, 0xee }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x68, 0xd0, 0xb8, 0xa0, 0xc8, 0x70, 0x18, + 0x40, 0x28, 0x90, 0xf8, 0xe0, 0x88, 0x30, 0x58 }, + { 0x00, 0x4e, 0x81, 0xcf, 0x1f, 0x51, 0x9e, 0xd0, + 0x3e, 0x70, 0xbf, 0xf1, 0x21, 0x6f, 0xa0, 0xee }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, + 0x48, 0x21, 0x9a, 0xf3, 0xec, 0x85, 0x3e, 0x57 }, + { 0x00, 0x4e, 0x81, 0xcf, 0x1f, 0x51, 0x9e, 0xd0, + 0x23, 0x6d, 0xa2, 0xec, 0x3c, 0x72, 0xbd, 0xf3 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x6a, 0xd4, 0xbe, 0xa8, 0xc2, 0x7c, 0x16, + 0x50, 0x3a, 0x84, 0xee, 0xf8, 0x92, 0x2c, 0x46 }, + { 0x00, 0x4e, 0x81, 0xcf, 0x1f, 0x51, 0x9e, 0xd0, + 0x23, 0x6d, 0xa2, 0xec, 0x3c, 0x72, 0xbd, 0xf3 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x6b, 0xd6, 0xbd, 0xac, 0xc7, 0x7a, 0x11, + 0x58, 0x33, 0x8e, 0xe5, 0xf4, 0x9f, 0x22, 0x49 }, + { 0x00, 0x4e, 0x81, 0xcf, 0x02, 0x4c, 0x83, 0xcd, + 0x04, 0x4a, 0x85, 0xcb, 0x06, 0x48, 0x87, 0xc9 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x6c, 0xd8, 0xb4, 0xb0, 0xdc, 0x68, 0x04, + 0x60, 0x0c, 0xb8, 0xd4, 0xd0, 0xbc, 0x08, 0x64 }, + { 0x00, 0x4e, 0x81, 0xcf, 0x02, 0x4c, 0x83, 0xcd, + 0x04, 0x4a, 0x85, 0xcb, 0x06, 0x48, 0x87, 0xc9 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x6d, 0xda, 0xb7, 0xb4, 0xd9, 0x6e, 0x03, + 0x68, 0x05, 0xb2, 0xdf, 0xdc, 0xb1, 0x06, 0x6b }, + { 0x00, 0x4e, 0x81, 0xcf, 0x02, 0x4c, 0x83, 0xcd, + 0x19, 0x57, 0x98, 0xd6, 0x1b, 0x55, 0x9a, 0xd4 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x6e, 0xdc, 0xb2, 0xb8, 0xd6, 0x64, 0x0a, + 0x70, 0x1e, 0xac, 0xc2, 0xc8, 0xa6, 0x14, 0x7a }, + { 0x00, 0x4e, 0x81, 0xcf, 0x02, 0x4c, 0x83, 0xcd, + 0x19, 0x57, 0x98, 0xd6, 0x1b, 0x55, 0x9a, 0xd4 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x6f, 0xde, 0xb1, 0xbc, 0xd3, 0x62, 0x0d, + 0x78, 0x17, 0xa6, 0xc9, 0xc4, 0xab, 0x1a, 0x75 }, + { 0x00, 0x53, 0xa6, 0xf5, 0x51, 0x02, 0xf7, 0xa4, + 0xa2, 0xf1, 0x04, 0x57, 0xf3, 0xa0, 0x55, 0x06 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x53, 0xa6, 0xf5, 0x51, 0x02, 0xf7, 0xa4, + 0xa2, 0xf1, 0x04, 0x57, 0xf3, 0xa0, 0x55, 0x06 }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x71, 0xe2, 0x93, 0xc4, 0xb5, 0x26, 0x57, + 0x88, 0xf9, 0x6a, 0x1b, 0x4c, 0x3d, 0xae, 0xdf }, + { 0x00, 0x53, 0xa6, 0xf5, 0x51, 0x02, 0xf7, 0xa4, + 0xbf, 0xec, 0x19, 0x4a, 0xee, 0xbd, 0x48, 0x1b }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x72, 0xe4, 0x96, 0xc8, 0xba, 0x2c, 0x5e, + 0x90, 0xe2, 0x74, 0x06, 0x58, 0x2a, 0xbc, 0xce }, + { 0x00, 0x53, 0xa6, 0xf5, 0x51, 0x02, 0xf7, 0xa4, + 0xbf, 0xec, 0x19, 0x4a, 0xee, 0xbd, 0x48, 0x1b }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x73, 0xe6, 0x95, 0xcc, 0xbf, 0x2a, 0x59, + 0x98, 0xeb, 0x7e, 0x0d, 0x54, 0x27, 0xb2, 0xc1 }, + { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, + 0x98, 0xcb, 0x3e, 0x6d, 0xd4, 0x87, 0x72, 0x21 }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, + 0xa0, 0xd4, 0x48, 0x3c, 0x70, 0x04, 0x98, 0xec }, + { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, + 0x98, 0xcb, 0x3e, 0x6d, 0xd4, 0x87, 0x72, 0x21 }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x75, 0xea, 0x9f, 0xd4, 0xa1, 0x3e, 0x4b, + 0xa8, 0xdd, 0x42, 0x37, 0x7c, 0x09, 0x96, 0xe3 }, + { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, + 0x85, 0xd6, 0x23, 0x70, 0xc9, 0x9a, 0x6f, 0x3c }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x76, 0xec, 0x9a, 0xd8, 0xae, 0x34, 0x42, + 0xb0, 0xc6, 0x5c, 0x2a, 0x68, 0x1e, 0x84, 0xf2 }, + { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, + 0x85, 0xd6, 0x23, 0x70, 0xc9, 0x9a, 0x6f, 0x3c }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x77, 0xee, 0x99, 0xdc, 0xab, 0x32, 0x45, + 0xb8, 0xcf, 0x56, 0x21, 0x64, 0x13, 0x8a, 0xfd }, + { 0x00, 0x53, 0xbb, 0xe8, 0x6b, 0x38, 0xd0, 0x83, + 0xd6, 0x85, 0x6d, 0x3e, 0xbd, 0xee, 0x06, 0x55 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x78, 0xf0, 0x88, 0xe0, 0x98, 0x10, 0x68, + 0xc0, 0xb8, 0x30, 0x48, 0x20, 0x58, 0xd0, 0xa8 }, + { 0x00, 0x53, 0xbb, 0xe8, 0x6b, 0x38, 0xd0, 0x83, + 0xd6, 0x85, 0x6d, 0x3e, 0xbd, 0xee, 0x06, 0x55 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x79, 0xf2, 0x8b, 0xe4, 0x9d, 0x16, 0x6f, + 0xc8, 0xb1, 0x3a, 0x43, 0x2c, 0x55, 0xde, 0xa7 }, + { 0x00, 0x53, 0xbb, 0xe8, 0x6b, 0x38, 0xd0, 0x83, + 0xcb, 0x98, 0x70, 0x23, 0xa0, 0xf3, 0x1b, 0x48 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x7a, 0xf4, 0x8e, 0xe8, 0x92, 0x1c, 0x66, + 0xd0, 0xaa, 0x24, 0x5e, 0x38, 0x42, 0xcc, 0xb6 }, + { 0x00, 0x53, 0xbb, 0xe8, 0x6b, 0x38, 0xd0, 0x83, + 0xcb, 0x98, 0x70, 0x23, 0xa0, 0xf3, 0x1b, 0x48 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x7b, 0xf6, 0x8d, 0xec, 0x97, 0x1a, 0x61, + 0xd8, 0xa3, 0x2e, 0x55, 0x34, 0x4f, 0xc2, 0xb9 }, + { 0x00, 0x53, 0xbb, 0xe8, 0x76, 0x25, 0xcd, 0x9e, + 0xec, 0xbf, 0x57, 0x04, 0x9a, 0xc9, 0x21, 0x72 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x7c, 0xf8, 0x84, 0xf0, 0x8c, 0x08, 0x74, + 0xe0, 0x9c, 0x18, 0x64, 0x10, 0x6c, 0xe8, 0x94 }, + { 0x00, 0x53, 0xbb, 0xe8, 0x76, 0x25, 0xcd, 0x9e, + 0xec, 0xbf, 0x57, 0x04, 0x9a, 0xc9, 0x21, 0x72 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x7d, 0xfa, 0x87, 0xf4, 0x89, 0x0e, 0x73, + 0xe8, 0x95, 0x12, 0x6f, 0x1c, 0x61, 0xe6, 0x9b }, + { 0x00, 0x53, 0xbb, 0xe8, 0x76, 0x25, 0xcd, 0x9e, + 0xf1, 0xa2, 0x4a, 0x19, 0x87, 0xd4, 0x3c, 0x6f }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x7e, 0xfc, 0x82, 0xf8, 0x86, 0x04, 0x7a, + 0xf0, 0x8e, 0x0c, 0x72, 0x08, 0x76, 0xf4, 0x8a }, + { 0x00, 0x53, 0xbb, 0xe8, 0x76, 0x25, 0xcd, 0x9e, + 0xf1, 0xa2, 0x4a, 0x19, 0x87, 0xd4, 0x3c, 0x6f }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, + 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, + { 0x00, 0x7f, 0xfe, 0x81, 0xfc, 0x83, 0x02, 0x7d, + 0xf8, 0x87, 0x06, 0x79, 0x04, 0x7b, 0xfa, 0x85 }, + { 0x00, 0xe8, 0xcd, 0x25, 0x87, 0x6f, 0x4a, 0xa2, + 0x13, 0xfb, 0xde, 0x36, 0x94, 0x7c, 0x59, 0xb1 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0xe8, 0xcd, 0x25, 0x87, 0x6f, 0x4a, 0xa2, + 0x13, 0xfb, 0xde, 0x36, 0x94, 0x7c, 0x59, 0xb1 }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, + 0x08, 0x89, 0x0a, 0x8b, 0x0c, 0x8d, 0x0e, 0x8f }, + { 0x00, 0xe8, 0xcd, 0x25, 0x87, 0x6f, 0x4a, 0xa2, + 0x0e, 0xe6, 0xc3, 0x2b, 0x89, 0x61, 0x44, 0xac }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x82, 0x04, 0x86, 0x08, 0x8a, 0x0c, 0x8e, + 0x10, 0x92, 0x14, 0x96, 0x18, 0x9a, 0x1c, 0x9e }, + { 0x00, 0xe8, 0xcd, 0x25, 0x87, 0x6f, 0x4a, 0xa2, + 0x0e, 0xe6, 0xc3, 0x2b, 0x89, 0x61, 0x44, 0xac }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x83, 0x06, 0x85, 0x0c, 0x8f, 0x0a, 0x89, + 0x18, 0x9b, 0x1e, 0x9d, 0x14, 0x97, 0x12, 0x91 }, + { 0x00, 0xe8, 0xcd, 0x25, 0x9a, 0x72, 0x57, 0xbf, + 0x29, 0xc1, 0xe4, 0x0c, 0xb3, 0x5b, 0x7e, 0x96 }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x84, 0x08, 0x8c, 0x10, 0x94, 0x18, 0x9c, + 0x20, 0xa4, 0x28, 0xac, 0x30, 0xb4, 0x38, 0xbc }, + { 0x00, 0xe8, 0xcd, 0x25, 0x9a, 0x72, 0x57, 0xbf, + 0x29, 0xc1, 0xe4, 0x0c, 0xb3, 0x5b, 0x7e, 0x96 }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x85, 0x0a, 0x8f, 0x14, 0x91, 0x1e, 0x9b, + 0x28, 0xad, 0x22, 0xa7, 0x3c, 0xb9, 0x36, 0xb3 }, + { 0x00, 0xe8, 0xcd, 0x25, 0x9a, 0x72, 0x57, 0xbf, + 0x34, 0xdc, 0xf9, 0x11, 0xae, 0x46, 0x63, 0x8b }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x86, 0x0c, 0x8a, 0x18, 0x9e, 0x14, 0x92, + 0x30, 0xb6, 0x3c, 0xba, 0x28, 0xae, 0x24, 0xa2 }, + { 0x00, 0xe8, 0xcd, 0x25, 0x9a, 0x72, 0x57, 0xbf, + 0x34, 0xdc, 0xf9, 0x11, 0xae, 0x46, 0x63, 0x8b }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x87, 0x0e, 0x89, 0x1c, 0x9b, 0x12, 0x95, + 0x38, 0xbf, 0x36, 0xb1, 0x24, 0xa3, 0x2a, 0xad }, + { 0x00, 0xe8, 0xd0, 0x38, 0xbd, 0x55, 0x6d, 0x85, + 0x67, 0x8f, 0xb7, 0x5f, 0xda, 0x32, 0x0a, 0xe2 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x88, 0x10, 0x98, 0x20, 0xa8, 0x30, 0xb8, + 0x40, 0xc8, 0x50, 0xd8, 0x60, 0xe8, 0x70, 0xf8 }, + { 0x00, 0xe8, 0xd0, 0x38, 0xbd, 0x55, 0x6d, 0x85, + 0x67, 0x8f, 0xb7, 0x5f, 0xda, 0x32, 0x0a, 0xe2 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x89, 0x12, 0x9b, 0x24, 0xad, 0x36, 0xbf, + 0x48, 0xc1, 0x5a, 0xd3, 0x6c, 0xe5, 0x7e, 0xf7 }, + { 0x00, 0xe8, 0xd0, 0x38, 0xbd, 0x55, 0x6d, 0x85, + 0x7a, 0x92, 0xaa, 0x42, 0xc7, 0x2f, 0x17, 0xff }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x8a, 0x14, 0x9e, 0x28, 0xa2, 0x3c, 0xb6, + 0x50, 0xda, 0x44, 0xce, 0x78, 0xf2, 0x6c, 0xe6 }, + { 0x00, 0xe8, 0xd0, 0x38, 0xbd, 0x55, 0x6d, 0x85, + 0x7a, 0x92, 0xaa, 0x42, 0xc7, 0x2f, 0x17, 0xff }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x8b, 0x16, 0x9d, 0x2c, 0xa7, 0x3a, 0xb1, + 0x58, 0xd3, 0x4e, 0xc5, 0x74, 0xff, 0x62, 0xe9 }, + { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, + 0x5d, 0xb5, 0x8d, 0x65, 0xfd, 0x15, 0x2d, 0xc5 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x8c, 0x18, 0x94, 0x30, 0xbc, 0x28, 0xa4, + 0x60, 0xec, 0x78, 0xf4, 0x50, 0xdc, 0x48, 0xc4 }, + { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, + 0x5d, 0xb5, 0x8d, 0x65, 0xfd, 0x15, 0x2d, 0xc5 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x8d, 0x1a, 0x97, 0x34, 0xb9, 0x2e, 0xa3, + 0x68, 0xe5, 0x72, 0xff, 0x5c, 0xd1, 0x46, 0xcb }, + { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, + 0x40, 0xa8, 0x90, 0x78, 0xe0, 0x08, 0x30, 0xd8 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x8e, 0x1c, 0x92, 0x38, 0xb6, 0x24, 0xaa, + 0x70, 0xfe, 0x6c, 0xe2, 0x48, 0xc6, 0x54, 0xda }, + { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, + 0x40, 0xa8, 0x90, 0x78, 0xe0, 0x08, 0x30, 0xd8 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x8f, 0x1e, 0x91, 0x3c, 0xb3, 0x22, 0xad, + 0x78, 0xf7, 0x66, 0xe9, 0x44, 0xcb, 0x5a, 0xd5 }, + { 0x00, 0xf5, 0xf7, 0x02, 0xf3, 0x06, 0x04, 0xf1, + 0xfb, 0x0e, 0x0c, 0xf9, 0x08, 0xfd, 0xff, 0x0a }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0xf5, 0xf7, 0x02, 0xf3, 0x06, 0x04, 0xf1, + 0xfb, 0x0e, 0x0c, 0xf9, 0x08, 0xfd, 0xff, 0x0a }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x91, 0x22, 0xb3, 0x44, 0xd5, 0x66, 0xf7, + 0x88, 0x19, 0xaa, 0x3b, 0xcc, 0x5d, 0xee, 0x7f }, + { 0x00, 0xf5, 0xf7, 0x02, 0xf3, 0x06, 0x04, 0xf1, + 0xe6, 0x13, 0x11, 0xe4, 0x15, 0xe0, 0xe2, 0x17 }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x92, 0x24, 0xb6, 0x48, 0xda, 0x6c, 0xfe, + 0x90, 0x02, 0xb4, 0x26, 0xd8, 0x4a, 0xfc, 0x6e }, + { 0x00, 0xf5, 0xf7, 0x02, 0xf3, 0x06, 0x04, 0xf1, + 0xe6, 0x13, 0x11, 0xe4, 0x15, 0xe0, 0xe2, 0x17 }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x93, 0x26, 0xb5, 0x4c, 0xdf, 0x6a, 0xf9, + 0x98, 0x0b, 0xbe, 0x2d, 0xd4, 0x47, 0xf2, 0x61 }, + { 0x00, 0xf5, 0xf7, 0x02, 0xee, 0x1b, 0x19, 0xec, + 0xc1, 0x34, 0x36, 0xc3, 0x2f, 0xda, 0xd8, 0x2d }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x94, 0x28, 0xbc, 0x50, 0xc4, 0x78, 0xec, + 0xa0, 0x34, 0x88, 0x1c, 0xf0, 0x64, 0xd8, 0x4c }, + { 0x00, 0xf5, 0xf7, 0x02, 0xee, 0x1b, 0x19, 0xec, + 0xc1, 0x34, 0x36, 0xc3, 0x2f, 0xda, 0xd8, 0x2d }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x95, 0x2a, 0xbf, 0x54, 0xc1, 0x7e, 0xeb, + 0xa8, 0x3d, 0x82, 0x17, 0xfc, 0x69, 0xd6, 0x43 }, + { 0x00, 0xf5, 0xf7, 0x02, 0xee, 0x1b, 0x19, 0xec, + 0xdc, 0x29, 0x2b, 0xde, 0x32, 0xc7, 0xc5, 0x30 }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x96, 0x2c, 0xba, 0x58, 0xce, 0x74, 0xe2, + 0xb0, 0x26, 0x9c, 0x0a, 0xe8, 0x7e, 0xc4, 0x52 }, + { 0x00, 0xf5, 0xf7, 0x02, 0xee, 0x1b, 0x19, 0xec, + 0xdc, 0x29, 0x2b, 0xde, 0x32, 0xc7, 0xc5, 0x30 }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x97, 0x2e, 0xb9, 0x5c, 0xcb, 0x72, 0xe5, + 0xb8, 0x2f, 0x96, 0x01, 0xe4, 0x73, 0xca, 0x5d }, + { 0x00, 0xf5, 0xea, 0x1f, 0xc9, 0x3c, 0x23, 0xd6, + 0x8f, 0x7a, 0x65, 0x90, 0x46, 0xb3, 0xac, 0x59 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x98, 0x30, 0xa8, 0x60, 0xf8, 0x50, 0xc8, + 0xc0, 0x58, 0xf0, 0x68, 0xa0, 0x38, 0x90, 0x08 }, + { 0x00, 0xf5, 0xea, 0x1f, 0xc9, 0x3c, 0x23, 0xd6, + 0x8f, 0x7a, 0x65, 0x90, 0x46, 0xb3, 0xac, 0x59 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x99, 0x32, 0xab, 0x64, 0xfd, 0x56, 0xcf, + 0xc8, 0x51, 0xfa, 0x63, 0xac, 0x35, 0x9e, 0x07 }, + { 0x00, 0xf5, 0xea, 0x1f, 0xc9, 0x3c, 0x23, 0xd6, + 0x92, 0x67, 0x78, 0x8d, 0x5b, 0xae, 0xb1, 0x44 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x9a, 0x34, 0xae, 0x68, 0xf2, 0x5c, 0xc6, + 0xd0, 0x4a, 0xe4, 0x7e, 0xb8, 0x22, 0x8c, 0x16 }, + { 0x00, 0xf5, 0xea, 0x1f, 0xc9, 0x3c, 0x23, 0xd6, + 0x92, 0x67, 0x78, 0x8d, 0x5b, 0xae, 0xb1, 0x44 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x9b, 0x36, 0xad, 0x6c, 0xf7, 0x5a, 0xc1, + 0xd8, 0x43, 0xee, 0x75, 0xb4, 0x2f, 0x82, 0x19 }, + { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, + 0xb5, 0x40, 0x5f, 0xaa, 0x61, 0x94, 0x8b, 0x7e }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, + 0xe0, 0x7c, 0xd8, 0x44, 0x90, 0x0c, 0xa8, 0x34 }, + { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, + 0xb5, 0x40, 0x5f, 0xaa, 0x61, 0x94, 0x8b, 0x7e }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x9d, 0x3a, 0xa7, 0x74, 0xe9, 0x4e, 0xd3, + 0xe8, 0x75, 0xd2, 0x4f, 0x9c, 0x01, 0xa6, 0x3b }, + { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, + 0xa8, 0x5d, 0x42, 0xb7, 0x7c, 0x89, 0x96, 0x63 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x9e, 0x3c, 0xa2, 0x78, 0xe6, 0x44, 0xda, + 0xf0, 0x6e, 0xcc, 0x52, 0x88, 0x16, 0xb4, 0x2a }, + { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, + 0xa8, 0x5d, 0x42, 0xb7, 0x7c, 0x89, 0x96, 0x63 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, + { 0x00, 0x9f, 0x3e, 0xa1, 0x7c, 0xe3, 0x42, 0xdd, + 0xf8, 0x67, 0xc6, 0x59, 0x84, 0x1b, 0xba, 0x25 }, + { 0x00, 0xd2, 0xb9, 0x6b, 0x6f, 0xbd, 0xd6, 0x04, + 0xde, 0x0c, 0x67, 0xb5, 0xb1, 0x63, 0x08, 0xda }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0xd2, 0xb9, 0x6b, 0x6f, 0xbd, 0xd6, 0x04, + 0xde, 0x0c, 0x67, 0xb5, 0xb1, 0x63, 0x08, 0xda }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa1, 0x42, 0xe3, 0x84, 0x25, 0xc6, 0x67, + 0x08, 0xa9, 0x4a, 0xeb, 0x8c, 0x2d, 0xce, 0x6f }, + { 0x00, 0xd2, 0xb9, 0x6b, 0x6f, 0xbd, 0xd6, 0x04, + 0xc3, 0x11, 0x7a, 0xa8, 0xac, 0x7e, 0x15, 0xc7 }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa2, 0x44, 0xe6, 0x88, 0x2a, 0xcc, 0x6e, + 0x10, 0xb2, 0x54, 0xf6, 0x98, 0x3a, 0xdc, 0x7e }, + { 0x00, 0xd2, 0xb9, 0x6b, 0x6f, 0xbd, 0xd6, 0x04, + 0xc3, 0x11, 0x7a, 0xa8, 0xac, 0x7e, 0x15, 0xc7 }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa3, 0x46, 0xe5, 0x8c, 0x2f, 0xca, 0x69, + 0x18, 0xbb, 0x5e, 0xfd, 0x94, 0x37, 0xd2, 0x71 }, + { 0x00, 0xd2, 0xb9, 0x6b, 0x72, 0xa0, 0xcb, 0x19, + 0xe4, 0x36, 0x5d, 0x8f, 0x96, 0x44, 0x2f, 0xfd }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa4, 0x48, 0xec, 0x90, 0x34, 0xd8, 0x7c, + 0x20, 0x84, 0x68, 0xcc, 0xb0, 0x14, 0xf8, 0x5c }, + { 0x00, 0xd2, 0xb9, 0x6b, 0x72, 0xa0, 0xcb, 0x19, + 0xe4, 0x36, 0x5d, 0x8f, 0x96, 0x44, 0x2f, 0xfd }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa5, 0x4a, 0xef, 0x94, 0x31, 0xde, 0x7b, + 0x28, 0x8d, 0x62, 0xc7, 0xbc, 0x19, 0xf6, 0x53 }, + { 0x00, 0xd2, 0xb9, 0x6b, 0x72, 0xa0, 0xcb, 0x19, + 0xf9, 0x2b, 0x40, 0x92, 0x8b, 0x59, 0x32, 0xe0 }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, + 0x30, 0x96, 0x7c, 0xda, 0xa8, 0x0e, 0xe4, 0x42 }, + { 0x00, 0xd2, 0xb9, 0x6b, 0x72, 0xa0, 0xcb, 0x19, + 0xf9, 0x2b, 0x40, 0x92, 0x8b, 0x59, 0x32, 0xe0 }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa7, 0x4e, 0xe9, 0x9c, 0x3b, 0xd2, 0x75, + 0x38, 0x9f, 0x76, 0xd1, 0xa4, 0x03, 0xea, 0x4d }, + { 0x00, 0xd2, 0xa4, 0x76, 0x55, 0x87, 0xf1, 0x23, + 0xaa, 0x78, 0x0e, 0xdc, 0xff, 0x2d, 0x5b, 0x89 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa8, 0x50, 0xf8, 0xa0, 0x08, 0xf0, 0x58, + 0x40, 0xe8, 0x10, 0xb8, 0xe0, 0x48, 0xb0, 0x18 }, + { 0x00, 0xd2, 0xa4, 0x76, 0x55, 0x87, 0xf1, 0x23, + 0xaa, 0x78, 0x0e, 0xdc, 0xff, 0x2d, 0x5b, 0x89 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xa9, 0x52, 0xfb, 0xa4, 0x0d, 0xf6, 0x5f, + 0x48, 0xe1, 0x1a, 0xb3, 0xec, 0x45, 0xbe, 0x17 }, + { 0x00, 0xd2, 0xa4, 0x76, 0x55, 0x87, 0xf1, 0x23, + 0xb7, 0x65, 0x13, 0xc1, 0xe2, 0x30, 0x46, 0x94 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xaa, 0x54, 0xfe, 0xa8, 0x02, 0xfc, 0x56, + 0x50, 0xfa, 0x04, 0xae, 0xf8, 0x52, 0xac, 0x06 }, + { 0x00, 0xd2, 0xa4, 0x76, 0x55, 0x87, 0xf1, 0x23, + 0xb7, 0x65, 0x13, 0xc1, 0xe2, 0x30, 0x46, 0x94 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xab, 0x56, 0xfd, 0xac, 0x07, 0xfa, 0x51, + 0x58, 0xf3, 0x0e, 0xa5, 0xf4, 0x5f, 0xa2, 0x09 }, + { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, + 0x90, 0x42, 0x34, 0xe6, 0xd8, 0x0a, 0x7c, 0xae }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xac, 0x58, 0xf4, 0xb0, 0x1c, 0xe8, 0x44, + 0x60, 0xcc, 0x38, 0x94, 0xd0, 0x7c, 0x88, 0x24 }, + { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, + 0x90, 0x42, 0x34, 0xe6, 0xd8, 0x0a, 0x7c, 0xae }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xad, 0x5a, 0xf7, 0xb4, 0x19, 0xee, 0x43, + 0x68, 0xc5, 0x32, 0x9f, 0xdc, 0x71, 0x86, 0x2b }, + { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, + 0x8d, 0x5f, 0x29, 0xfb, 0xc5, 0x17, 0x61, 0xb3 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xae, 0x5c, 0xf2, 0xb8, 0x16, 0xe4, 0x4a, + 0x70, 0xde, 0x2c, 0x82, 0xc8, 0x66, 0x94, 0x3a }, + { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, + 0x8d, 0x5f, 0x29, 0xfb, 0xc5, 0x17, 0x61, 0xb3 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xaf, 0x5e, 0xf1, 0xbc, 0x13, 0xe2, 0x4d, + 0x78, 0xd7, 0x26, 0x89, 0xc4, 0x6b, 0x9a, 0x35 }, + { 0x00, 0xcf, 0x83, 0x4c, 0x1b, 0xd4, 0x98, 0x57, + 0x36, 0xf9, 0xb5, 0x7a, 0x2d, 0xe2, 0xae, 0x61 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0xcf, 0x83, 0x4c, 0x1b, 0xd4, 0x98, 0x57, + 0x36, 0xf9, 0xb5, 0x7a, 0x2d, 0xe2, 0xae, 0x61 }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb1, 0x62, 0xd3, 0xc4, 0x75, 0xa6, 0x17, + 0x88, 0x39, 0xea, 0x5b, 0x4c, 0xfd, 0x2e, 0x9f }, + { 0x00, 0xcf, 0x83, 0x4c, 0x1b, 0xd4, 0x98, 0x57, + 0x2b, 0xe4, 0xa8, 0x67, 0x30, 0xff, 0xb3, 0x7c }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb2, 0x64, 0xd6, 0xc8, 0x7a, 0xac, 0x1e, + 0x90, 0x22, 0xf4, 0x46, 0x58, 0xea, 0x3c, 0x8e }, + { 0x00, 0xcf, 0x83, 0x4c, 0x1b, 0xd4, 0x98, 0x57, + 0x2b, 0xe4, 0xa8, 0x67, 0x30, 0xff, 0xb3, 0x7c }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb3, 0x66, 0xd5, 0xcc, 0x7f, 0xaa, 0x19, + 0x98, 0x2b, 0xfe, 0x4d, 0x54, 0xe7, 0x32, 0x81 }, + { 0x00, 0xcf, 0x83, 0x4c, 0x06, 0xc9, 0x85, 0x4a, + 0x0c, 0xc3, 0x8f, 0x40, 0x0a, 0xc5, 0x89, 0x46 }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb4, 0x68, 0xdc, 0xd0, 0x64, 0xb8, 0x0c, + 0xa0, 0x14, 0xc8, 0x7c, 0x70, 0xc4, 0x18, 0xac }, + { 0x00, 0xcf, 0x83, 0x4c, 0x06, 0xc9, 0x85, 0x4a, + 0x0c, 0xc3, 0x8f, 0x40, 0x0a, 0xc5, 0x89, 0x46 }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb5, 0x6a, 0xdf, 0xd4, 0x61, 0xbe, 0x0b, + 0xa8, 0x1d, 0xc2, 0x77, 0x7c, 0xc9, 0x16, 0xa3 }, + { 0x00, 0xcf, 0x83, 0x4c, 0x06, 0xc9, 0x85, 0x4a, + 0x11, 0xde, 0x92, 0x5d, 0x17, 0xd8, 0x94, 0x5b }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb6, 0x6c, 0xda, 0xd8, 0x6e, 0xb4, 0x02, + 0xb0, 0x06, 0xdc, 0x6a, 0x68, 0xde, 0x04, 0xb2 }, + { 0x00, 0xcf, 0x83, 0x4c, 0x06, 0xc9, 0x85, 0x4a, + 0x11, 0xde, 0x92, 0x5d, 0x17, 0xd8, 0x94, 0x5b }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb7, 0x6e, 0xd9, 0xdc, 0x6b, 0xb2, 0x05, + 0xb8, 0x0f, 0xd6, 0x61, 0x64, 0xd3, 0x0a, 0xbd }, + { 0x00, 0xcf, 0x9e, 0x51, 0x21, 0xee, 0xbf, 0x70, + 0x42, 0x8d, 0xdc, 0x13, 0x63, 0xac, 0xfd, 0x32 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb8, 0x70, 0xc8, 0xe0, 0x58, 0x90, 0x28, + 0xc0, 0x78, 0xb0, 0x08, 0x20, 0x98, 0x50, 0xe8 }, + { 0x00, 0xcf, 0x9e, 0x51, 0x21, 0xee, 0xbf, 0x70, + 0x42, 0x8d, 0xdc, 0x13, 0x63, 0xac, 0xfd, 0x32 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xb9, 0x72, 0xcb, 0xe4, 0x5d, 0x96, 0x2f, + 0xc8, 0x71, 0xba, 0x03, 0x2c, 0x95, 0x5e, 0xe7 }, + { 0x00, 0xcf, 0x9e, 0x51, 0x21, 0xee, 0xbf, 0x70, + 0x5f, 0x90, 0xc1, 0x0e, 0x7e, 0xb1, 0xe0, 0x2f }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xba, 0x74, 0xce, 0xe8, 0x52, 0x9c, 0x26, + 0xd0, 0x6a, 0xa4, 0x1e, 0x38, 0x82, 0x4c, 0xf6 }, + { 0x00, 0xcf, 0x9e, 0x51, 0x21, 0xee, 0xbf, 0x70, + 0x5f, 0x90, 0xc1, 0x0e, 0x7e, 0xb1, 0xe0, 0x2f }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, + 0xd8, 0x63, 0xae, 0x15, 0x34, 0x8f, 0x42, 0xf9 }, + { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, + 0x78, 0xb7, 0xe6, 0x29, 0x44, 0x8b, 0xda, 0x15 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xbc, 0x78, 0xc4, 0xf0, 0x4c, 0x88, 0x34, + 0xe0, 0x5c, 0x98, 0x24, 0x10, 0xac, 0x68, 0xd4 }, + { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, + 0x78, 0xb7, 0xe6, 0x29, 0x44, 0x8b, 0xda, 0x15 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xbd, 0x7a, 0xc7, 0xf4, 0x49, 0x8e, 0x33, + 0xe8, 0x55, 0x92, 0x2f, 0x1c, 0xa1, 0x66, 0xdb }, + { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, + 0x65, 0xaa, 0xfb, 0x34, 0x59, 0x96, 0xc7, 0x08 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xbe, 0x7c, 0xc2, 0xf8, 0x46, 0x84, 0x3a, + 0xf0, 0x4e, 0x8c, 0x32, 0x08, 0xb6, 0x74, 0xca }, + { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, + 0x65, 0xaa, 0xfb, 0x34, 0x59, 0x96, 0xc7, 0x08 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, + 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, + { 0x00, 0xbf, 0x7e, 0xc1, 0xfc, 0x43, 0x82, 0x3d, + 0xf8, 0x47, 0x86, 0x39, 0x04, 0xbb, 0x7a, 0xc5 }, + { 0x00, 0x9c, 0x25, 0xb9, 0x4a, 0xd6, 0x6f, 0xf3, + 0x94, 0x08, 0xb1, 0x2d, 0xde, 0x42, 0xfb, 0x67 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x9c, 0x25, 0xb9, 0x4a, 0xd6, 0x6f, 0xf3, + 0x94, 0x08, 0xb1, 0x2d, 0xde, 0x42, 0xfb, 0x67 }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc1, 0x82, 0x43, 0x04, 0xc5, 0x86, 0x47, + 0x08, 0xc9, 0x8a, 0x4b, 0x0c, 0xcd, 0x8e, 0x4f }, + { 0x00, 0x9c, 0x25, 0xb9, 0x4a, 0xd6, 0x6f, 0xf3, + 0x89, 0x15, 0xac, 0x30, 0xc3, 0x5f, 0xe6, 0x7a }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc2, 0x84, 0x46, 0x08, 0xca, 0x8c, 0x4e, + 0x10, 0xd2, 0x94, 0x56, 0x18, 0xda, 0x9c, 0x5e }, + { 0x00, 0x9c, 0x25, 0xb9, 0x4a, 0xd6, 0x6f, 0xf3, + 0x89, 0x15, 0xac, 0x30, 0xc3, 0x5f, 0xe6, 0x7a }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc3, 0x86, 0x45, 0x0c, 0xcf, 0x8a, 0x49, + 0x18, 0xdb, 0x9e, 0x5d, 0x14, 0xd7, 0x92, 0x51 }, + { 0x00, 0x9c, 0x25, 0xb9, 0x57, 0xcb, 0x72, 0xee, + 0xae, 0x32, 0x8b, 0x17, 0xf9, 0x65, 0xdc, 0x40 }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc4, 0x88, 0x4c, 0x10, 0xd4, 0x98, 0x5c, + 0x20, 0xe4, 0xa8, 0x6c, 0x30, 0xf4, 0xb8, 0x7c }, + { 0x00, 0x9c, 0x25, 0xb9, 0x57, 0xcb, 0x72, 0xee, + 0xae, 0x32, 0x8b, 0x17, 0xf9, 0x65, 0xdc, 0x40 }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc5, 0x8a, 0x4f, 0x14, 0xd1, 0x9e, 0x5b, + 0x28, 0xed, 0xa2, 0x67, 0x3c, 0xf9, 0xb6, 0x73 }, + { 0x00, 0x9c, 0x25, 0xb9, 0x57, 0xcb, 0x72, 0xee, + 0xb3, 0x2f, 0x96, 0x0a, 0xe4, 0x78, 0xc1, 0x5d }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc6, 0x8c, 0x4a, 0x18, 0xde, 0x94, 0x52, + 0x30, 0xf6, 0xbc, 0x7a, 0x28, 0xee, 0xa4, 0x62 }, + { 0x00, 0x9c, 0x25, 0xb9, 0x57, 0xcb, 0x72, 0xee, + 0xb3, 0x2f, 0x96, 0x0a, 0xe4, 0x78, 0xc1, 0x5d }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc7, 0x8e, 0x49, 0x1c, 0xdb, 0x92, 0x55, + 0x38, 0xff, 0xb6, 0x71, 0x24, 0xe3, 0xaa, 0x6d }, + { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, + 0xe0, 0x7c, 0xd8, 0x44, 0x90, 0x0c, 0xa8, 0x34 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc8, 0x90, 0x58, 0x20, 0xe8, 0xb0, 0x78, + 0x40, 0x88, 0xd0, 0x18, 0x60, 0xa8, 0xf0, 0x38 }, + { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, + 0xe0, 0x7c, 0xd8, 0x44, 0x90, 0x0c, 0xa8, 0x34 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xc9, 0x92, 0x5b, 0x24, 0xed, 0xb6, 0x7f, + 0x48, 0x81, 0xda, 0x13, 0x6c, 0xa5, 0xfe, 0x37 }, + { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, + 0xfd, 0x61, 0xc5, 0x59, 0x8d, 0x11, 0xb5, 0x29 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xca, 0x94, 0x5e, 0x28, 0xe2, 0xbc, 0x76, + 0x50, 0x9a, 0xc4, 0x0e, 0x78, 0xb2, 0xec, 0x26 }, + { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, + 0xfd, 0x61, 0xc5, 0x59, 0x8d, 0x11, 0xb5, 0x29 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xcb, 0x96, 0x5d, 0x2c, 0xe7, 0xba, 0x71, + 0x58, 0x93, 0xce, 0x05, 0x74, 0xbf, 0xe2, 0x29 }, + { 0x00, 0x9c, 0x38, 0xa4, 0x6d, 0xf1, 0x55, 0xc9, + 0xda, 0x46, 0xe2, 0x7e, 0xb7, 0x2b, 0x8f, 0x13 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xcc, 0x98, 0x54, 0x30, 0xfc, 0xa8, 0x64, + 0x60, 0xac, 0xf8, 0x34, 0x50, 0x9c, 0xc8, 0x04 }, + { 0x00, 0x9c, 0x38, 0xa4, 0x6d, 0xf1, 0x55, 0xc9, + 0xda, 0x46, 0xe2, 0x7e, 0xb7, 0x2b, 0x8f, 0x13 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xcd, 0x9a, 0x57, 0x34, 0xf9, 0xae, 0x63, + 0x68, 0xa5, 0xf2, 0x3f, 0x5c, 0x91, 0xc6, 0x0b }, + { 0x00, 0x9c, 0x38, 0xa4, 0x6d, 0xf1, 0x55, 0xc9, + 0xc7, 0x5b, 0xff, 0x63, 0xaa, 0x36, 0x92, 0x0e }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xce, 0x9c, 0x52, 0x38, 0xf6, 0xa4, 0x6a, + 0x70, 0xbe, 0xec, 0x22, 0x48, 0x86, 0xd4, 0x1a }, + { 0x00, 0x9c, 0x38, 0xa4, 0x6d, 0xf1, 0x55, 0xc9, + 0xc7, 0x5b, 0xff, 0x63, 0xaa, 0x36, 0x92, 0x0e }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, + 0x78, 0xb7, 0xe6, 0x29, 0x44, 0x8b, 0xda, 0x15 }, + { 0x00, 0x81, 0x1f, 0x9e, 0x3e, 0xbf, 0x21, 0xa0, + 0x7c, 0xfd, 0x63, 0xe2, 0x42, 0xc3, 0x5d, 0xdc }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x81, 0x1f, 0x9e, 0x3e, 0xbf, 0x21, 0xa0, + 0x7c, 0xfd, 0x63, 0xe2, 0x42, 0xc3, 0x5d, 0xdc }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd1, 0xa2, 0x73, 0x44, 0x95, 0xe6, 0x37, + 0x88, 0x59, 0x2a, 0xfb, 0xcc, 0x1d, 0x6e, 0xbf }, + { 0x00, 0x81, 0x1f, 0x9e, 0x3e, 0xbf, 0x21, 0xa0, + 0x61, 0xe0, 0x7e, 0xff, 0x5f, 0xde, 0x40, 0xc1 }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, + 0x90, 0x42, 0x34, 0xe6, 0xd8, 0x0a, 0x7c, 0xae }, + { 0x00, 0x81, 0x1f, 0x9e, 0x3e, 0xbf, 0x21, 0xa0, + 0x61, 0xe0, 0x7e, 0xff, 0x5f, 0xde, 0x40, 0xc1 }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd3, 0xa6, 0x75, 0x4c, 0x9f, 0xea, 0x39, + 0x98, 0x4b, 0x3e, 0xed, 0xd4, 0x07, 0x72, 0xa1 }, + { 0x00, 0x81, 0x1f, 0x9e, 0x23, 0xa2, 0x3c, 0xbd, + 0x46, 0xc7, 0x59, 0xd8, 0x65, 0xe4, 0x7a, 0xfb }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd4, 0xa8, 0x7c, 0x50, 0x84, 0xf8, 0x2c, + 0xa0, 0x74, 0x08, 0xdc, 0xf0, 0x24, 0x58, 0x8c }, + { 0x00, 0x81, 0x1f, 0x9e, 0x23, 0xa2, 0x3c, 0xbd, + 0x46, 0xc7, 0x59, 0xd8, 0x65, 0xe4, 0x7a, 0xfb }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd5, 0xaa, 0x7f, 0x54, 0x81, 0xfe, 0x2b, + 0xa8, 0x7d, 0x02, 0xd7, 0xfc, 0x29, 0x56, 0x83 }, + { 0x00, 0x81, 0x1f, 0x9e, 0x23, 0xa2, 0x3c, 0xbd, + 0x5b, 0xda, 0x44, 0xc5, 0x78, 0xf9, 0x67, 0xe6 }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd6, 0xac, 0x7a, 0x58, 0x8e, 0xf4, 0x22, + 0xb0, 0x66, 0x1c, 0xca, 0xe8, 0x3e, 0x44, 0x92 }, + { 0x00, 0x81, 0x1f, 0x9e, 0x23, 0xa2, 0x3c, 0xbd, + 0x5b, 0xda, 0x44, 0xc5, 0x78, 0xf9, 0x67, 0xe6 }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd7, 0xae, 0x79, 0x5c, 0x8b, 0xf2, 0x25, + 0xb8, 0x6f, 0x16, 0xc1, 0xe4, 0x33, 0x4a, 0x9d }, + { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, + 0x08, 0x89, 0x0a, 0x8b, 0x0c, 0x8d, 0x0e, 0x8f }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd8, 0xb0, 0x68, 0x60, 0xb8, 0xd0, 0x08, + 0xc0, 0x18, 0x70, 0xa8, 0xa0, 0x78, 0x10, 0xc8 }, + { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, + 0x08, 0x89, 0x0a, 0x8b, 0x0c, 0x8d, 0x0e, 0x8f }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xd9, 0xb2, 0x6b, 0x64, 0xbd, 0xd6, 0x0f, + 0xc8, 0x11, 0x7a, 0xa3, 0xac, 0x75, 0x1e, 0xc7 }, + { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, + 0x15, 0x94, 0x17, 0x96, 0x11, 0x90, 0x13, 0x92 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xda, 0xb4, 0x6e, 0x68, 0xb2, 0xdc, 0x06, + 0xd0, 0x0a, 0x64, 0xbe, 0xb8, 0x62, 0x0c, 0xd6 }, + { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, + 0x15, 0x94, 0x17, 0x96, 0x11, 0x90, 0x13, 0x92 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xdb, 0xb6, 0x6d, 0x6c, 0xb7, 0xda, 0x01, + 0xd8, 0x03, 0x6e, 0xb5, 0xb4, 0x6f, 0x02, 0xd9 }, + { 0x00, 0x81, 0x02, 0x83, 0x19, 0x98, 0x1b, 0x9a, + 0x32, 0xb3, 0x30, 0xb1, 0x2b, 0xaa, 0x29, 0xa8 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xdc, 0xb8, 0x64, 0x70, 0xac, 0xc8, 0x14, + 0xe0, 0x3c, 0x58, 0x84, 0x90, 0x4c, 0x28, 0xf4 }, + { 0x00, 0x81, 0x02, 0x83, 0x19, 0x98, 0x1b, 0x9a, + 0x32, 0xb3, 0x30, 0xb1, 0x2b, 0xaa, 0x29, 0xa8 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xdd, 0xba, 0x67, 0x74, 0xa9, 0xce, 0x13, + 0xe8, 0x35, 0x52, 0x8f, 0x9c, 0x41, 0x26, 0xfb }, + { 0x00, 0x81, 0x02, 0x83, 0x19, 0x98, 0x1b, 0x9a, + 0x2f, 0xae, 0x2d, 0xac, 0x36, 0xb7, 0x34, 0xb5 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xde, 0xbc, 0x62, 0x78, 0xa6, 0xc4, 0x1a, + 0xf0, 0x2e, 0x4c, 0x92, 0x88, 0x56, 0x34, 0xea }, + { 0x00, 0x81, 0x02, 0x83, 0x19, 0x98, 0x1b, 0x9a, + 0x2f, 0xae, 0x2d, 0xac, 0x36, 0xb7, 0x34, 0xb5 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, + { 0x00, 0xdf, 0xbe, 0x61, 0x7c, 0xa3, 0xc2, 0x1d, + 0xf8, 0x27, 0x46, 0x99, 0x84, 0x5b, 0x3a, 0xe5 }, + { 0x00, 0xa6, 0x51, 0xf7, 0xa2, 0x04, 0xf3, 0x55, + 0x59, 0xff, 0x08, 0xae, 0xfb, 0x5d, 0xaa, 0x0c }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0xa6, 0x51, 0xf7, 0xa2, 0x04, 0xf3, 0x55, + 0x59, 0xff, 0x08, 0xae, 0xfb, 0x5d, 0xaa, 0x0c }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe1, 0xc2, 0x23, 0x84, 0x65, 0x46, 0xa7, + 0x08, 0xe9, 0xca, 0x2b, 0x8c, 0x6d, 0x4e, 0xaf }, + { 0x00, 0xa6, 0x51, 0xf7, 0xa2, 0x04, 0xf3, 0x55, + 0x44, 0xe2, 0x15, 0xb3, 0xe6, 0x40, 0xb7, 0x11 }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe2, 0xc4, 0x26, 0x88, 0x6a, 0x4c, 0xae, + 0x10, 0xf2, 0xd4, 0x36, 0x98, 0x7a, 0x5c, 0xbe }, + { 0x00, 0xa6, 0x51, 0xf7, 0xa2, 0x04, 0xf3, 0x55, + 0x44, 0xe2, 0x15, 0xb3, 0xe6, 0x40, 0xb7, 0x11 }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe3, 0xc6, 0x25, 0x8c, 0x6f, 0x4a, 0xa9, + 0x18, 0xfb, 0xde, 0x3d, 0x94, 0x77, 0x52, 0xb1 }, + { 0x00, 0xa6, 0x51, 0xf7, 0xbf, 0x19, 0xee, 0x48, + 0x63, 0xc5, 0x32, 0x94, 0xdc, 0x7a, 0x8d, 0x2b }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe4, 0xc8, 0x2c, 0x90, 0x74, 0x58, 0xbc, + 0x20, 0xc4, 0xe8, 0x0c, 0xb0, 0x54, 0x78, 0x9c }, + { 0x00, 0xa6, 0x51, 0xf7, 0xbf, 0x19, 0xee, 0x48, + 0x63, 0xc5, 0x32, 0x94, 0xdc, 0x7a, 0x8d, 0x2b }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe5, 0xca, 0x2f, 0x94, 0x71, 0x5e, 0xbb, + 0x28, 0xcd, 0xe2, 0x07, 0xbc, 0x59, 0x76, 0x93 }, + { 0x00, 0xa6, 0x51, 0xf7, 0xbf, 0x19, 0xee, 0x48, + 0x7e, 0xd8, 0x2f, 0x89, 0xc1, 0x67, 0x90, 0x36 }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe6, 0xcc, 0x2a, 0x98, 0x7e, 0x54, 0xb2, + 0x30, 0xd6, 0xfc, 0x1a, 0xa8, 0x4e, 0x64, 0x82 }, + { 0x00, 0xa6, 0x51, 0xf7, 0xbf, 0x19, 0xee, 0x48, + 0x7e, 0xd8, 0x2f, 0x89, 0xc1, 0x67, 0x90, 0x36 }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe7, 0xce, 0x29, 0x9c, 0x7b, 0x52, 0xb5, + 0x38, 0xdf, 0xf6, 0x11, 0xa4, 0x43, 0x6a, 0x8d }, + { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, + 0x2d, 0x8b, 0x61, 0xc7, 0xb5, 0x13, 0xf9, 0x5f }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, + 0x40, 0xa8, 0x90, 0x78, 0xe0, 0x08, 0x30, 0xd8 }, + { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, + 0x2d, 0x8b, 0x61, 0xc7, 0xb5, 0x13, 0xf9, 0x5f }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xe9, 0xd2, 0x3b, 0xa4, 0x4d, 0x76, 0x9f, + 0x48, 0xa1, 0x9a, 0x73, 0xec, 0x05, 0x3e, 0xd7 }, + { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, + 0x30, 0x96, 0x7c, 0xda, 0xa8, 0x0e, 0xe4, 0x42 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xea, 0xd4, 0x3e, 0xa8, 0x42, 0x7c, 0x96, + 0x50, 0xba, 0x84, 0x6e, 0xf8, 0x12, 0x2c, 0xc6 }, + { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, + 0x30, 0x96, 0x7c, 0xda, 0xa8, 0x0e, 0xe4, 0x42 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xeb, 0xd6, 0x3d, 0xac, 0x47, 0x7a, 0x91, + 0x58, 0xb3, 0x8e, 0x65, 0xf4, 0x1f, 0x22, 0xc9 }, + { 0x00, 0xa6, 0x4c, 0xea, 0x85, 0x23, 0xc9, 0x6f, + 0x17, 0xb1, 0x5b, 0xfd, 0x92, 0x34, 0xde, 0x78 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xec, 0xd8, 0x34, 0xb0, 0x5c, 0x68, 0x84, + 0x60, 0x8c, 0xb8, 0x54, 0xd0, 0x3c, 0x08, 0xe4 }, + { 0x00, 0xa6, 0x4c, 0xea, 0x85, 0x23, 0xc9, 0x6f, + 0x17, 0xb1, 0x5b, 0xfd, 0x92, 0x34, 0xde, 0x78 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xed, 0xda, 0x37, 0xb4, 0x59, 0x6e, 0x83, + 0x68, 0x85, 0xb2, 0x5f, 0xdc, 0x31, 0x06, 0xeb }, + { 0x00, 0xa6, 0x4c, 0xea, 0x85, 0x23, 0xc9, 0x6f, + 0x0a, 0xac, 0x46, 0xe0, 0x8f, 0x29, 0xc3, 0x65 }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xee, 0xdc, 0x32, 0xb8, 0x56, 0x64, 0x8a, + 0x70, 0x9e, 0xac, 0x42, 0xc8, 0x26, 0x14, 0xfa }, + { 0x00, 0xa6, 0x4c, 0xea, 0x85, 0x23, 0xc9, 0x6f, + 0x0a, 0xac, 0x46, 0xe0, 0x8f, 0x29, 0xc3, 0x65 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xef, 0xde, 0x31, 0xbc, 0x53, 0x62, 0x8d, + 0x78, 0x97, 0xa6, 0x49, 0xc4, 0x2b, 0x1a, 0xf5 }, + { 0x00, 0xbb, 0x6b, 0xd0, 0xd6, 0x6d, 0xbd, 0x06, + 0xb1, 0x0a, 0xda, 0x61, 0x67, 0xdc, 0x0c, 0xb7 }, + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0xbb, 0x6b, 0xd0, 0xd6, 0x6d, 0xbd, 0x06, + 0xb1, 0x0a, 0xda, 0x61, 0x67, 0xdc, 0x0c, 0xb7 }, + { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, + 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf1, 0xe2, 0x13, 0xc4, 0x35, 0x26, 0xd7, + 0x88, 0x79, 0x6a, 0x9b, 0x4c, 0xbd, 0xae, 0x5f }, + { 0x00, 0xbb, 0x6b, 0xd0, 0xd6, 0x6d, 0xbd, 0x06, + 0xac, 0x17, 0xc7, 0x7c, 0x7a, 0xc1, 0x11, 0xaa }, + { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, + 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf2, 0xe4, 0x16, 0xc8, 0x3a, 0x2c, 0xde, + 0x90, 0x62, 0x74, 0x86, 0x58, 0xaa, 0xbc, 0x4e }, + { 0x00, 0xbb, 0x6b, 0xd0, 0xd6, 0x6d, 0xbd, 0x06, + 0xac, 0x17, 0xc7, 0x7c, 0x7a, 0xc1, 0x11, 0xaa }, + { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, + 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf3, 0xe6, 0x15, 0xcc, 0x3f, 0x2a, 0xd9, + 0x98, 0x6b, 0x7e, 0x8d, 0x54, 0xa7, 0xb2, 0x41 }, + { 0x00, 0xbb, 0x6b, 0xd0, 0xcb, 0x70, 0xa0, 0x1b, + 0x8b, 0x30, 0xe0, 0x5b, 0x40, 0xfb, 0x2b, 0x90 }, + { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, + 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf4, 0xe8, 0x1c, 0xd0, 0x24, 0x38, 0xcc, + 0xa0, 0x54, 0x48, 0xbc, 0x70, 0x84, 0x98, 0x6c }, + { 0x00, 0xbb, 0x6b, 0xd0, 0xcb, 0x70, 0xa0, 0x1b, + 0x8b, 0x30, 0xe0, 0x5b, 0x40, 0xfb, 0x2b, 0x90 }, + { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, + 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, + 0xa8, 0x5d, 0x42, 0xb7, 0x7c, 0x89, 0x96, 0x63 }, + { 0x00, 0xbb, 0x6b, 0xd0, 0xcb, 0x70, 0xa0, 0x1b, + 0x96, 0x2d, 0xfd, 0x46, 0x5d, 0xe6, 0x36, 0x8d }, + { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, + 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf6, 0xec, 0x1a, 0xd8, 0x2e, 0x34, 0xc2, + 0xb0, 0x46, 0x5c, 0xaa, 0x68, 0x9e, 0x84, 0x72 }, + { 0x00, 0xbb, 0x6b, 0xd0, 0xcb, 0x70, 0xa0, 0x1b, + 0x96, 0x2d, 0xfd, 0x46, 0x5d, 0xe6, 0x36, 0x8d }, + { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, + 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf7, 0xee, 0x19, 0xdc, 0x2b, 0x32, 0xc5, + 0xb8, 0x4f, 0x56, 0xa1, 0x64, 0x93, 0x8a, 0x7d }, + { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, + 0xc5, 0x7e, 0xb3, 0x08, 0x29, 0x92, 0x5f, 0xe4 }, + { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, + 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf8, 0xf0, 0x08, 0xe0, 0x18, 0x10, 0xe8, + 0xc0, 0x38, 0x30, 0xc8, 0x20, 0xd8, 0xd0, 0x28 }, + { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, + 0xc5, 0x7e, 0xb3, 0x08, 0x29, 0x92, 0x5f, 0xe4 }, + { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, + 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xf9, 0xf2, 0x0b, 0xe4, 0x1d, 0x16, 0xef, + 0xc8, 0x31, 0x3a, 0xc3, 0x2c, 0xd5, 0xde, 0x27 }, + { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, + 0xd8, 0x63, 0xae, 0x15, 0x34, 0x8f, 0x42, 0xf9 }, + { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, + 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xfa, 0xf4, 0x0e, 0xe8, 0x12, 0x1c, 0xe6, + 0xd0, 0x2a, 0x24, 0xde, 0x38, 0xc2, 0xcc, 0x36 }, + { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, + 0xd8, 0x63, 0xae, 0x15, 0x34, 0x8f, 0x42, 0xf9 }, + { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, + 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xfb, 0xf6, 0x0d, 0xec, 0x17, 0x1a, 0xe1, + 0xd8, 0x23, 0x2e, 0xd5, 0x34, 0xcf, 0xc2, 0x39 }, + { 0x00, 0xbb, 0x76, 0xcd, 0xf1, 0x4a, 0x87, 0x3c, + 0xff, 0x44, 0x89, 0x32, 0x0e, 0xb5, 0x78, 0xc3 }, + { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, + 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xfc, 0xf8, 0x04, 0xf0, 0x0c, 0x08, 0xf4, + 0xe0, 0x1c, 0x18, 0xe4, 0x10, 0xec, 0xe8, 0x14 }, + { 0x00, 0xbb, 0x76, 0xcd, 0xf1, 0x4a, 0x87, 0x3c, + 0xff, 0x44, 0x89, 0x32, 0x0e, 0xb5, 0x78, 0xc3 }, + { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, + 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xfd, 0xfa, 0x07, 0xf4, 0x09, 0x0e, 0xf3, + 0xe8, 0x15, 0x12, 0xef, 0x1c, 0xe1, 0xe6, 0x1b }, + { 0x00, 0xbb, 0x76, 0xcd, 0xf1, 0x4a, 0x87, 0x3c, + 0xe2, 0x59, 0x94, 0x2f, 0x13, 0xa8, 0x65, 0xde }, + { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, + 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xfe, 0xfc, 0x02, 0xf8, 0x06, 0x04, 0xfa, + 0xf0, 0x0e, 0x0c, 0xf2, 0x08, 0xf6, 0xf4, 0x0a }, + { 0x00, 0xbb, 0x76, 0xcd, 0xf1, 0x4a, 0x87, 0x3c, + 0xe2, 0x59, 0x94, 0x2f, 0x13, 0xa8, 0x65, 0xde }, + { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, + 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, + { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, + 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, + { 0x00, 0xff, 0xfe, 0x01, 0xfc, 0x03, 0x02, 0xfd, + 0xf8, 0x07, 0x06, 0xf9, 0x04, 0xfb, 0xfa, 0x05 } +}; + +#endif /* defined(__x86_64) && (defined(HAVE_SSSE3) || defined(HAVE_AVX2)) */ diff --git a/module/zfs/zap.c b/module/zfs/zap.c index 454b4be62ed3..9e4f05049b21 100644 --- a/module/zfs/zap.c +++ b/module/zfs/zap.c @@ -967,10 +967,18 @@ fzap_prefetch(zap_name_t *zn) uint64_t zap_create_link(objset_t *os, dmu_object_type_t ot, uint64_t parent_obj, const char *name, dmu_tx_t *tx) +{ + return (zap_create_link_dnsize(os, ot, parent_obj, name, 0, tx)); +} + +uint64_t +zap_create_link_dnsize(objset_t *os, dmu_object_type_t ot, uint64_t parent_obj, + const char *name, int dnodesize, dmu_tx_t *tx) { uint64_t new_obj; - VERIFY((new_obj = zap_create(os, ot, DMU_OT_NONE, 0, tx)) > 0); + VERIFY((new_obj = zap_create_dnsize(os, ot, DMU_OT_NONE, 0, + dnodesize, tx)) > 0); VERIFY0(zap_add(os, parent_obj, name, sizeof (uint64_t), 1, &new_obj, tx)); diff --git a/module/zfs/zap_micro.c b/module/zfs/zap_micro.c index 3faf27ce3646..f3153cc18133 100644 --- a/module/zfs/zap_micro.c +++ b/module/zfs/zap_micro.c @@ -630,18 +630,36 @@ int zap_create_claim(objset_t *os, uint64_t obj, dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) { - return (zap_create_claim_norm(os, obj, - 0, ot, bonustype, bonuslen, tx)); + return (zap_create_claim_dnsize(os, obj, ot, bonustype, bonuslen, + 0, tx)); +} + +int +zap_create_claim_dnsize(objset_t *os, uint64_t obj, dmu_object_type_t ot, + dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx) +{ + return (zap_create_claim_norm_dnsize(os, obj, + 0, ot, bonustype, bonuslen, dnodesize, tx)); } int zap_create_claim_norm(objset_t *os, uint64_t obj, int normflags, dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) +{ + return (zap_create_claim_norm_dnsize(os, obj, normflags, ot, bonustype, + bonuslen, 0, tx)); +} + +int +zap_create_claim_norm_dnsize(objset_t *os, uint64_t obj, int normflags, + dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, + int dnodesize, dmu_tx_t *tx) { int err; - err = dmu_object_claim(os, obj, ot, 0, bonustype, bonuslen, tx); + err = dmu_object_claim_dnsize(os, obj, ot, 0, bonustype, bonuslen, + dnodesize, tx); if (err != 0) return (err); mzap_create_impl(os, obj, normflags, 0, tx); @@ -655,11 +673,28 @@ zap_create(objset_t *os, dmu_object_type_t ot, return (zap_create_norm(os, 0, ot, bonustype, bonuslen, tx)); } +uint64_t +zap_create_dnsize(objset_t *os, dmu_object_type_t ot, + dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx) +{ + return (zap_create_norm_dnsize(os, 0, ot, bonustype, bonuslen, + dnodesize, tx)); +} + uint64_t zap_create_norm(objset_t *os, int normflags, dmu_object_type_t ot, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) { - uint64_t obj = dmu_object_alloc(os, ot, 0, bonustype, bonuslen, tx); + return (zap_create_norm_dnsize(os, normflags, ot, bonustype, bonuslen, + 0, tx)); +} + +uint64_t +zap_create_norm_dnsize(objset_t *os, int normflags, dmu_object_type_t ot, + dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx) +{ + uint64_t obj = dmu_object_alloc_dnsize(os, ot, 0, bonustype, bonuslen, + dnodesize, tx); mzap_create_impl(os, obj, normflags, 0, tx); return (obj); @@ -670,7 +705,17 @@ zap_create_flags(objset_t *os, int normflags, zap_flags_t flags, dmu_object_type_t ot, int leaf_blockshift, int indirect_blockshift, dmu_object_type_t bonustype, int bonuslen, dmu_tx_t *tx) { - uint64_t obj = dmu_object_alloc(os, ot, 0, bonustype, bonuslen, tx); + return (zap_create_flags_dnsize(os, normflags, flags, ot, + leaf_blockshift, indirect_blockshift, bonustype, bonuslen, 0, tx)); +} + +uint64_t +zap_create_flags_dnsize(objset_t *os, int normflags, zap_flags_t flags, + dmu_object_type_t ot, int leaf_blockshift, int indirect_blockshift, + dmu_object_type_t bonustype, int bonuslen, int dnodesize, dmu_tx_t *tx) +{ + uint64_t obj = dmu_object_alloc_dnsize(os, ot, 0, bonustype, bonuslen, + dnodesize, tx); ASSERT(leaf_blockshift >= SPA_MINBLOCKSHIFT && leaf_blockshift <= SPA_OLD_MAXBLOCKSHIFT && @@ -1458,10 +1503,14 @@ zap_count_write(objset_t *os, uint64_t zapobj, const char *name, int add, #if defined(_KERNEL) && defined(HAVE_SPL) EXPORT_SYMBOL(zap_create); +EXPORT_SYMBOL(zap_create_dnsize); EXPORT_SYMBOL(zap_create_norm); +EXPORT_SYMBOL(zap_create_norm_dnsize); EXPORT_SYMBOL(zap_create_flags); +EXPORT_SYMBOL(zap_create_flags_dnsize); EXPORT_SYMBOL(zap_create_claim); EXPORT_SYMBOL(zap_create_claim_norm); +EXPORT_SYMBOL(zap_create_claim_norm_dnsize); EXPORT_SYMBOL(zap_destroy); EXPORT_SYMBOL(zap_lookup); EXPORT_SYMBOL(zap_lookup_norm); diff --git a/module/zfs/zfeature_common.c b/module/zfs/zfeature_common.c index f57e5489cae9..3264f6235274 100644 --- a/module/zfs/zfeature_common.c +++ b/module/zfs/zfeature_common.c @@ -242,4 +242,15 @@ zpool_feature_init(void) "Support for blocks larger than 128KB.", ZFEATURE_FLAG_PER_DATASET, large_blocks_deps); } + + { + static const spa_feature_t large_dnode_deps[] = { + SPA_FEATURE_EXTENSIBLE_DATASET, + SPA_FEATURE_NONE + }; + zfeature_register(SPA_FEATURE_LARGE_DNODE, + "org.zfsonlinux:large_dnode", "large_dnode", + "Variable on-disk size of dnodes.", + ZFEATURE_FLAG_PER_DATASET, large_dnode_deps); + } } diff --git a/module/zfs/zfs_acl.c b/module/zfs/zfs_acl.c index 47cfd464b164..bbb731495634 100644 --- a/module/zfs/zfs_acl.c +++ b/module/zfs/zfs_acl.c @@ -53,6 +53,7 @@ #include #include #include +#include #include "fs/fs_subr.h" #define ALLOW ACE_ACCESS_ALLOWED_ACE_TYPE @@ -1166,7 +1167,8 @@ zfs_acl_chown_setattr(znode_t *zp) error = zfs_acl_node_read(zp, B_TRUE, &aclp, B_FALSE); if (error == 0 && aclp->z_acl_count > 0) zp->z_mode = zfs_mode_compute(zp->z_mode, aclp, - &zp->z_pflags, zp->z_uid, zp->z_gid); + &zp->z_pflags, KUID_TO_SUID(ZTOI(zp)->i_uid), + KGID_TO_SGID(ZTOI(zp)->i_gid)); /* * Some ZFS implementations (ZEVO) create neither a ZNODE_ACL @@ -1324,7 +1326,7 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx) mode = zp->z_mode; mode = zfs_mode_compute(mode, aclp, &zp->z_pflags, - zp->z_uid, zp->z_gid); + zfs_uid_read(ZTOI(zp)), zfs_gid_read(ZTOI(zp))); zp->z_mode = mode; SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zsb), NULL, @@ -1394,7 +1396,7 @@ zfs_aclset_common(znode_t *zp, zfs_acl_t *aclp, cred_t *cr, dmu_tx_t *tx) otype == DMU_OT_ACL ? DMU_OT_SYSACL : DMU_OT_NONE, otype == DMU_OT_ACL ? - DN_MAX_BONUSLEN : 0, tx); + DN_OLD_MAX_BONUSLEN : 0, tx); } else { (void) dmu_object_set_blocksize(zsb->z_os, aoid, aclp->z_acl_bytes, 0, tx); @@ -1744,9 +1746,7 @@ zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr, int error; zfs_sb_t *zsb = ZTOZSB(dzp); zfs_acl_t *paclp; -#ifdef HAVE_KSID - gid_t gid; -#endif /* HAVE_KSID */ + gid_t gid = vap->va_gid; boolean_t need_chmod = B_TRUE; boolean_t inherited = B_FALSE; @@ -1780,7 +1780,7 @@ zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr, (uint64_t)vap->va_gid, cr, ZFS_GROUP, &acl_ids->z_fuidp); gid = vap->va_gid; - if (acl_ids->z_fgid != dzp->z_gid && + if (acl_ids->z_fgid != KGID_TO_SGID(ZTOI(dzp)->i_gid) && !groupmember(vap->va_gid, cr) && secpolicy_vnode_create_gid(cr) != 0) acl_ids->z_fgid = 0; @@ -1790,7 +1790,8 @@ zfs_acl_ids_create(znode_t *dzp, int flag, vattr_t *vap, cred_t *cr, char *domain; uint32_t rid; - acl_ids->z_fgid = dzp->z_gid; + acl_ids->z_fgid = KGID_TO_SGID( + ZTOI(dzp)->i_gid); gid = zfs_fuid_map_id(zsb, acl_ids->z_fgid, cr, ZFS_GROUP); @@ -2342,7 +2343,8 @@ zfs_has_access(znode_t *zp, cred_t *cr) if (zfs_zaccess_aces_check(zp, &have, B_TRUE, cr) != 0) { uid_t owner; - owner = zfs_fuid_map_id(ZTOZSB(zp), zp->z_uid, cr, ZFS_OWNER); + owner = zfs_fuid_map_id(ZTOZSB(zp), + KUID_TO_SUID(ZTOI(zp)->i_uid), cr, ZFS_OWNER); return (secpolicy_vnode_any_access(cr, ZTOI(zp), owner) == 0); } return (B_TRUE); @@ -2420,12 +2422,13 @@ zfs_fastaccesschk_execute(znode_t *zdp, cred_t *cr) return (0); } - if (FUID_INDEX(zdp->z_uid) != 0 || FUID_INDEX(zdp->z_gid) != 0) { + if (KUID_TO_SUID(ZTOI(zdp)->i_uid) != 0 || + KGID_TO_SGID(ZTOI(zdp)->i_gid) != 0) { mutex_exit(&zdp->z_acl_lock); goto slow; } - if (uid == zdp->z_uid) { + if (uid == KUID_TO_SUID(ZTOI(zdp)->i_uid)) { owner = B_TRUE; if (zdp->z_mode & S_IXUSR) { mutex_exit(&zdp->z_acl_lock); @@ -2435,7 +2438,7 @@ zfs_fastaccesschk_execute(znode_t *zdp, cred_t *cr) goto slow; } } - if (groupmember(zdp->z_gid, cr)) { + if (groupmember(KGID_TO_SGID(ZTOI(zdp)->i_gid), cr)) { groupmbr = B_TRUE; if (zdp->z_mode & S_IXGRP) { mutex_exit(&zdp->z_acl_lock); @@ -2473,53 +2476,33 @@ zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr) { uint32_t working_mode; int error; - boolean_t check_privs; - znode_t *check_zp = zp; + int is_attr; + boolean_t check_privs; + znode_t *xzp; + znode_t *check_zp = zp; mode_t needed_bits; uid_t owner; + is_attr = ((zp->z_pflags & ZFS_XATTR) && S_ISDIR(ZTOI(zp)->i_mode)); + /* * If attribute then validate against base file */ - if ((zp->z_pflags & ZFS_XATTR) && S_ISDIR(ZTOI(zp)->i_mode)) { + if (is_attr) { uint64_t parent; - rw_enter(&zp->z_xattr_lock, RW_READER); - if (zp->z_xattr_parent) { - check_zp = zp->z_xattr_parent; - rw_exit(&zp->z_xattr_lock); - - /* - * Verify a lookup yields the same znode. - */ - ASSERT3S(sa_lookup(zp->z_sa_hdl, SA_ZPL_PARENT( - ZTOZSB(zp)), &parent, sizeof (parent)), ==, 0); - ASSERT3U(check_zp->z_id, ==, parent); - } else { - rw_exit(&zp->z_xattr_lock); - - error = sa_lookup(zp->z_sa_hdl, SA_ZPL_PARENT( - ZTOZSB(zp)), &parent, sizeof (parent)); - if (error) - return (error); + if ((error = sa_lookup(zp->z_sa_hdl, + SA_ZPL_PARENT(ZTOZSB(zp)), &parent, + sizeof (parent))) != 0) + return (error); - /* - * Cache the lookup on the parent file znode as - * zp->z_xattr_parent and hold a reference. This - * effectively pins the parent in memory until all - * child xattr znodes have been destroyed and - * release their references in zfs_inode_destroy(). - */ - error = zfs_zget(ZTOZSB(zp), parent, &check_zp); - if (error) - return (error); - - rw_enter(&zp->z_xattr_lock, RW_WRITER); - if (zp->z_xattr_parent == NULL) - zp->z_xattr_parent = check_zp; - rw_exit(&zp->z_xattr_lock); + if ((error = zfs_zget(ZTOZSB(zp), + parent, &xzp)) != 0) { + return (error); } + check_zp = xzp; + /* * fixup mode to map to xattr perms */ @@ -2535,7 +2518,8 @@ zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr) } } - owner = zfs_fuid_map_id(ZTOZSB(zp), zp->z_uid, cr, ZFS_OWNER); + owner = zfs_fuid_map_id(ZTOZSB(zp), KUID_TO_SUID(ZTOI(zp)->i_uid), + cr, ZFS_OWNER); /* * Map the bits required to the standard inode flags * S_IRUSR|S_IWUSR|S_IXUSR in the needed_bits. Map the bits @@ -2561,11 +2545,15 @@ zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr) if ((error = zfs_zaccess_common(check_zp, mode, &working_mode, &check_privs, skipaclchk, cr)) == 0) { + if (is_attr) + iput(ZTOI(xzp)); return (secpolicy_vnode_access2(cr, ZTOI(zp), owner, needed_bits, needed_bits)); } if (error && !check_privs) { + if (is_attr) + iput(ZTOI(xzp)); return (error); } @@ -2626,6 +2614,9 @@ zfs_zaccess(znode_t *zp, int mode, int flags, boolean_t skipaclchk, cred_t *cr) needed_bits, needed_bits); } + if (is_attr) + iput(ZTOI(xzp)); + return (error); } @@ -2657,7 +2648,8 @@ zfs_delete_final_check(znode_t *zp, znode_t *dzp, int error; uid_t downer; - downer = zfs_fuid_map_id(ZTOZSB(dzp), dzp->z_uid, cr, ZFS_OWNER); + downer = zfs_fuid_map_id(ZTOZSB(dzp), KUID_TO_SUID(ZTOI(dzp)->i_uid), + cr, ZFS_OWNER); error = secpolicy_vnode_access2(cr, ZTOI(dzp), downer, available_perms, S_IWUSR|S_IXUSR); diff --git a/module/zfs/zfs_ctldir.c b/module/zfs/zfs_ctldir.c index 7d160f23d904..459f01c98721 100644 --- a/module/zfs/zfs_ctldir.c +++ b/module/zfs/zfs_ctldir.c @@ -109,7 +109,7 @@ static krwlock_t zfs_snapshot_lock; * Control Directory Tunables (.zfs) */ int zfs_expire_snapshot = ZFSCTL_EXPIRE_SNAPSHOT; -int zfs_admin_snapshot = 0; +int zfs_admin_snapshot = 1; /* * Dedicated task queue for unmounting snapshots. @@ -478,10 +478,7 @@ zfsctl_inode_alloc(zfs_sb_t *zsb, uint64_t id, zp->z_seq = 0; zp->z_mapcnt = 0; zp->z_size = 0; - zp->z_links = 0; zp->z_pflags = 0; - zp->z_uid = 0; - zp->z_gid = 0; zp->z_mode = 0; zp->z_sync_cnt = 0; zp->z_is_mapped = B_FALSE; @@ -490,7 +487,7 @@ zfsctl_inode_alloc(zfs_sb_t *zsb, uint64_t id, zp->z_is_stale = B_FALSE; ip->i_generation = 0; ip->i_ino = id; - ip->i_mode = (S_IFDIR | S_IRUGO | S_IXUGO); + ip->i_mode = (S_IFDIR | S_IRWXUGO); ip->i_uid = SUID_TO_KUID(0); ip->i_gid = SGID_TO_KGID(0); ip->i_blkbits = SPA_MINBLOCKSHIFT; @@ -749,12 +746,13 @@ zfsctl_snapshot_path_objset(zfs_sb_t *zsb, uint64_t objsetid, return (ENOENT); cookie = spl_fstrans_mark(); - snapname = kmem_alloc(MAXNAMELEN, KM_SLEEP); + snapname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); while (error == 0) { dsl_pool_config_enter(dmu_objset_pool(os), FTAG); - error = dmu_snapshot_list_next(zsb->z_os, MAXNAMELEN, - snapname, &id, &pos, &case_conflict); + error = dmu_snapshot_list_next(zsb->z_os, + ZFS_MAX_DATASET_NAME_LEN, snapname, &id, &pos, + &case_conflict); dsl_pool_config_exit(dmu_objset_pool(os), FTAG); if (error) goto out; @@ -767,7 +765,7 @@ zfsctl_snapshot_path_objset(zfs_sb_t *zsb, uint64_t objsetid, snprintf(full_path, path_len - 1, "%s/.zfs/snapshot/%s", zsb->z_mntopts->z_mntpoint, snapname); out: - kmem_free(snapname, MAXNAMELEN); + kmem_free(snapname, ZFS_MAX_DATASET_NAME_LEN); spl_fstrans_unmark(cookie); return (error); @@ -854,14 +852,14 @@ zfsctl_snapdir_rename(struct inode *sdip, char *snm, ZFS_ENTER(zsb); - to = kmem_alloc(MAXNAMELEN, KM_SLEEP); - from = kmem_alloc(MAXNAMELEN, KM_SLEEP); - real = kmem_alloc(MAXNAMELEN, KM_SLEEP); - fsname = kmem_alloc(MAXNAMELEN, KM_SLEEP); + to = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); + from = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); + real = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); + fsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); if (zsb->z_case == ZFS_CASE_INSENSITIVE) { error = dmu_snapshot_realname(zsb->z_os, snm, real, - MAXNAMELEN, NULL); + ZFS_MAX_DATASET_NAME_LEN, NULL); if (error == 0) { snm = real; } else if (error != ENOTSUP) { @@ -871,9 +869,11 @@ zfsctl_snapdir_rename(struct inode *sdip, char *snm, dmu_objset_name(zsb->z_os, fsname); - error = zfsctl_snapshot_name(ITOZSB(sdip), snm, MAXNAMELEN, from); + error = zfsctl_snapshot_name(ITOZSB(sdip), snm, + ZFS_MAX_DATASET_NAME_LEN, from); if (error == 0) - error = zfsctl_snapshot_name(ITOZSB(tdip), tnm, MAXNAMELEN, to); + error = zfsctl_snapshot_name(ITOZSB(tdip), tnm, + ZFS_MAX_DATASET_NAME_LEN, to); if (error == 0) error = zfs_secpolicy_rename_perms(from, to, cr); if (error != 0) @@ -903,10 +903,10 @@ zfsctl_snapdir_rename(struct inode *sdip, char *snm, rw_exit(&zfs_snapshot_lock); out: - kmem_free(from, MAXNAMELEN); - kmem_free(to, MAXNAMELEN); - kmem_free(real, MAXNAMELEN); - kmem_free(fsname, MAXNAMELEN); + kmem_free(from, ZFS_MAX_DATASET_NAME_LEN); + kmem_free(to, ZFS_MAX_DATASET_NAME_LEN); + kmem_free(real, ZFS_MAX_DATASET_NAME_LEN); + kmem_free(fsname, ZFS_MAX_DATASET_NAME_LEN); ZFS_EXIT(zsb); @@ -929,12 +929,12 @@ zfsctl_snapdir_remove(struct inode *dip, char *name, cred_t *cr, int flags) ZFS_ENTER(zsb); - snapname = kmem_alloc(MAXNAMELEN, KM_SLEEP); - real = kmem_alloc(MAXNAMELEN, KM_SLEEP); + snapname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); + real = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); if (zsb->z_case == ZFS_CASE_INSENSITIVE) { error = dmu_snapshot_realname(zsb->z_os, name, real, - MAXNAMELEN, NULL); + ZFS_MAX_DATASET_NAME_LEN, NULL); if (error == 0) { name = real; } else if (error != ENOTSUP) { @@ -942,7 +942,8 @@ zfsctl_snapdir_remove(struct inode *dip, char *name, cred_t *cr, int flags) } } - error = zfsctl_snapshot_name(ITOZSB(dip), name, MAXNAMELEN, snapname); + error = zfsctl_snapshot_name(ITOZSB(dip), name, + ZFS_MAX_DATASET_NAME_LEN, snapname); if (error == 0) error = zfs_secpolicy_destroy_perms(snapname, cr); if (error != 0) @@ -952,8 +953,8 @@ zfsctl_snapdir_remove(struct inode *dip, char *name, cred_t *cr, int flags) if ((error == 0) || (error == ENOENT)) error = dsl_destroy_snapshot(snapname, B_FALSE); out: - kmem_free(snapname, MAXNAMELEN); - kmem_free(real, MAXNAMELEN); + kmem_free(snapname, ZFS_MAX_DATASET_NAME_LEN); + kmem_free(real, ZFS_MAX_DATASET_NAME_LEN); ZFS_EXIT(zsb); @@ -975,7 +976,7 @@ zfsctl_snapdir_mkdir(struct inode *dip, char *dirname, vattr_t *vap, if (!zfs_admin_snapshot) return (EACCES); - dsname = kmem_alloc(MAXNAMELEN, KM_SLEEP); + dsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); if (zfs_component_namecheck(dirname, NULL, NULL) != 0) { error = SET_ERROR(EILSEQ); @@ -997,7 +998,7 @@ zfsctl_snapdir_mkdir(struct inode *dip, char *dirname, vattr_t *vap, 0, cr, NULL, NULL); } out: - kmem_free(dsname, MAXNAMELEN); + kmem_free(dsname, ZFS_MAX_DATASET_NAME_LEN); return (error); } @@ -1075,11 +1076,11 @@ zfsctl_snapshot_mount(struct path *path, int flags) zsb = ITOZSB(ip); ZFS_ENTER(zsb); - full_name = kmem_zalloc(MAXNAMELEN, KM_SLEEP); + full_name = kmem_zalloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); full_path = kmem_zalloc(MAXPATHLEN, KM_SLEEP); error = zfsctl_snapshot_name(zsb, dname(dentry), - MAXNAMELEN, full_name); + ZFS_MAX_DATASET_NAME_LEN, full_name); if (error) goto error; @@ -1153,7 +1154,7 @@ zfsctl_snapshot_mount(struct path *path, int flags) } path_put(&spath); error: - kmem_free(full_name, MAXNAMELEN); + kmem_free(full_name, ZFS_MAX_DATASET_NAME_LEN); kmem_free(full_path, MAXPATHLEN); ZFS_EXIT(zsb); diff --git a/module/zfs/zfs_dir.c b/module/zfs/zfs_dir.c index bdcb87b942f2..8eee626d9814 100644 --- a/module/zfs/zfs_dir.c +++ b/module/zfs/zfs_dir.c @@ -478,7 +478,7 @@ zfs_unlinked_add(znode_t *zp, dmu_tx_t *tx) zfs_sb_t *zsb = ZTOZSB(zp); ASSERT(zp->z_unlinked); - ASSERT(zp->z_links == 0); + ASSERT(ZTOI(zp)->i_nlink == 0); VERIFY3U(0, ==, zap_add_int(zsb->z_os, zsb->z_unlinkedobj, zp->z_id, tx)); @@ -612,9 +612,10 @@ zfs_rmnode(znode_t *zp) dmu_tx_t *tx; uint64_t acl_obj; uint64_t xattr_obj; + uint64_t links; int error; - ASSERT(zp->z_links == 0); + ASSERT(ZTOI(zp)->i_nlink == 0); ASSERT(atomic_read(&ZTOI(zp)->i_count) == 0); /* @@ -694,9 +695,10 @@ zfs_rmnode(znode_t *zp) ASSERT(error == 0); mutex_enter(&xzp->z_lock); xzp->z_unlinked = B_TRUE; /* mark xzp for deletion */ - xzp->z_links = 0; /* no more links to it */ + clear_nlink(ZTOI(xzp)); /* no more links to it */ + links = 0; VERIFY(0 == sa_update(xzp->z_sa_hdl, SA_ZPL_LINKS(zsb), - &xzp->z_links, sizeof (xzp->z_links), tx)); + &links, sizeof (links), tx)); mutex_exit(&xzp->z_lock); zfs_unlinked_add(xzp, tx); } @@ -735,6 +737,7 @@ zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag) int zp_is_dir = S_ISDIR(ZTOI(zp)->i_mode); sa_bulk_attr_t bulk[5]; uint64_t mtime[2], ctime[2]; + uint64_t links; int count = 0; int error; @@ -746,10 +749,16 @@ zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag) mutex_exit(&zp->z_lock); return (SET_ERROR(ENOENT)); } - zp->z_links++; - SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zsb), NULL, - &zp->z_links, sizeof (zp->z_links)); - + if (!(flag & ZNEW)) { + /* + * ZNEW nodes come from zfs_mknode() where the link + * count has already been initialised + */ + inc_nlink(ZTOI(zp)); + links = ZTOI(zp)->i_nlink; + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zsb), NULL, + &links, sizeof (links)); + } } SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zsb), NULL, &dzp->z_id, sizeof (dzp->z_id)); @@ -769,12 +778,14 @@ zfs_link_create(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag) mutex_enter(&dzp->z_lock); dzp->z_size++; - dzp->z_links += zp_is_dir; + if (zp_is_dir) + inc_nlink(ZTOI(dzp)); + links = ZTOI(dzp)->i_nlink; count = 0; SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zsb), NULL, &dzp->z_size, sizeof (dzp->z_size)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zsb), NULL, - &dzp->z_links, sizeof (dzp->z_links)); + &links, sizeof (links)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zsb), NULL, mtime, sizeof (mtime)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zsb), NULL, @@ -835,6 +846,7 @@ zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag, boolean_t unlinked = B_FALSE; sa_bulk_attr_t bulk[5]; uint64_t mtime[2], ctime[2]; + uint64_t links; int count = 0; int error; @@ -861,15 +873,16 @@ zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag, return (error); } - if (zp->z_links <= zp_is_dir) { + if (ZTOI(zp)->i_nlink <= zp_is_dir) { zfs_panic_recover("zfs: link count on %lu is %u, " "should be at least %u", zp->z_id, - (int)zp->z_links, zp_is_dir + 1); - zp->z_links = zp_is_dir + 1; + (int)ZTOI(zp)->i_nlink, zp_is_dir + 1); + set_nlink(ZTOI(zp), zp_is_dir + 1); } - if (--zp->z_links == zp_is_dir) { + drop_nlink(ZTOI(zp)); + if (ZTOI(zp)->i_nlink == zp_is_dir) { zp->z_unlinked = B_TRUE; - zp->z_links = 0; + clear_nlink(ZTOI(zp)); unlinked = B_TRUE; } else { SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zsb), @@ -879,8 +892,9 @@ zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag, zfs_tstamp_update_setup(zp, STATE_CHANGED, mtime, ctime); } + links = ZTOI(zp)->i_nlink; SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zsb), - NULL, &zp->z_links, sizeof (zp->z_links)); + NULL, &links, sizeof (links)); error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); count = 0; ASSERT(error == 0); @@ -893,9 +907,11 @@ zfs_link_destroy(zfs_dirlock_t *dl, znode_t *zp, dmu_tx_t *tx, int flag, mutex_enter(&dzp->z_lock); dzp->z_size--; /* one dirent removed */ - dzp->z_links -= zp_is_dir; /* ".." link from zp */ + if (zp_is_dir) + drop_nlink(ZTOI(dzp)); /* ".." link from zp */ + links = ZTOI(dzp)->i_nlink; SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zsb), - NULL, &dzp->z_links, sizeof (dzp->z_links)); + NULL, &links, sizeof (links)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zsb), NULL, &dzp->z_size, sizeof (dzp->z_size)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zsb), @@ -1088,8 +1104,10 @@ zfs_sticky_remove_access(znode_t *zdp, znode_t *zp, cred_t *cr) if ((zdp->z_mode & S_ISVTX) == 0) return (0); - downer = zfs_fuid_map_id(zsb, zdp->z_uid, cr, ZFS_OWNER); - fowner = zfs_fuid_map_id(zsb, zp->z_uid, cr, ZFS_OWNER); + downer = zfs_fuid_map_id(zsb, KUID_TO_SUID(ZTOI(zdp)->i_uid), + cr, ZFS_OWNER); + fowner = zfs_fuid_map_id(zsb, KUID_TO_SUID(ZTOI(zp)->i_uid), + cr, ZFS_OWNER); if ((uid = crgetuid(cr)) == downer || uid == fowner || (S_ISDIR(ZTOI(zp)->i_mode) && diff --git a/module/zfs/zfs_fuid.c b/module/zfs/zfs_fuid.c index 6ca61b87242f..eb0ffe419d16 100644 --- a/module/zfs/zfs_fuid.c +++ b/module/zfs/zfs_fuid.c @@ -387,8 +387,10 @@ zfs_fuid_find_by_idx(zfs_sb_t *zsb, uint32_t idx) void zfs_fuid_map_ids(znode_t *zp, cred_t *cr, uid_t *uidp, uid_t *gidp) { - *uidp = zfs_fuid_map_id(ZTOZSB(zp), zp->z_uid, cr, ZFS_OWNER); - *gidp = zfs_fuid_map_id(ZTOZSB(zp), zp->z_gid, cr, ZFS_GROUP); + *uidp = zfs_fuid_map_id(ZTOZSB(zp), KUID_TO_SUID(ZTOI(zp)->i_uid), + cr, ZFS_OWNER); + *gidp = zfs_fuid_map_id(ZTOZSB(zp), KGID_TO_SGID(ZTOI(zp)->i_gid), + cr, ZFS_GROUP); } uid_t diff --git a/module/zfs/zfs_ioctl.c b/module/zfs/zfs_ioctl.c index 7969f525ef6f..8e187d59ce99 100644 --- a/module/zfs/zfs_ioctl.c +++ b/module/zfs/zfs_ioctl.c @@ -22,9 +22,10 @@ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * Portions Copyright 2011 Martin Matuska + * Copyright 2015, OmniTI Computer Consulting, Inc. All rights reserved. * Portions Copyright 2012 Pawel Jakub Dawidek * Copyright (c) 2012, Joyent, Inc. All rights reserved. - * Copyright 2015 Nexenta Systems, Inc. All rights reserved. + * Copyright 2016 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2014, Joyent, Inc. All rights reserved. * Copyright (c) 2011, 2015 by Delphix. All rights reserved. * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. @@ -186,12 +187,19 @@ #include #include +#include #include "zfs_namecheck.h" #include "zfs_prop.h" #include "zfs_deleg.h" #include "zfs_comutil.h" +/* + * Limit maximum nvlist size. We don't want users passing in insane values + * for zc->zc_nvlist_src_size, since we will need to allocate that much memory. + */ +#define MAX_NVLIST_SRC_SIZE KMALLOC_MAX_SIZE + kmutex_t zfsdev_state_lock; zfsdev_state_t *zfsdev_state_list; @@ -596,7 +604,7 @@ zfs_secpolicy_setprop(const char *dsname, zfs_prop_t prop, nvpair_t *propval, case ZFS_PROP_SNAPSHOT_LIMIT: if (!INGLOBALZONE(curproc)) { uint64_t zoned; - char setpoint[MAXNAMELEN]; + char setpoint[ZFS_MAX_DATASET_NAME_LEN]; /* * Unprivileged users are allowed to modify the * limit on things *under* (ie. contained by) @@ -838,7 +846,7 @@ zfs_secpolicy_destroy_snaps(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) int zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr) { - char parentname[MAXNAMELEN]; + char parentname[ZFS_MAX_DATASET_NAME_LEN]; int error; if ((error = zfs_secpolicy_write_perms(from, @@ -891,7 +899,7 @@ zfs_secpolicy_promote(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) error = dsl_dataset_hold(dp, zc->zc_name, FTAG, &clone); if (error == 0) { - char parentname[MAXNAMELEN]; + char parentname[ZFS_MAX_DATASET_NAME_LEN]; dsl_dataset_t *origin = NULL; dsl_dir_t *dd; dd = clone->ds_dir; @@ -937,6 +945,13 @@ zfs_secpolicy_recv(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) ZFS_DELEG_PERM_CREATE, cr)); } +/* ARGSUSED */ +static int +zfs_secpolicy_recv_new(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) +{ + return (zfs_secpolicy_recv(zc, innvl, cr)); +} + int zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr) { @@ -1061,7 +1076,7 @@ zfs_secpolicy_log_history(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) static int zfs_secpolicy_create_clone(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) { - char parentname[MAXNAMELEN]; + char parentname[ZFS_MAX_DATASET_NAME_LEN]; int error; char *origin; @@ -1204,7 +1219,7 @@ zfs_secpolicy_hold(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) for (pair = nvlist_next_nvpair(holds, NULL); pair != NULL; pair = nvlist_next_nvpair(holds, pair)) { - char fsname[MAXNAMELEN]; + char fsname[ZFS_MAX_DATASET_NAME_LEN]; error = dmu_fsname(nvpair_name(pair), fsname); if (error != 0) return (error); @@ -1225,7 +1240,7 @@ zfs_secpolicy_release(zfs_cmd_t *zc, nvlist_t *innvl, cred_t *cr) for (pair = nvlist_next_nvpair(innvl, NULL); pair != NULL; pair = nvlist_next_nvpair(innvl, pair)) { - char fsname[MAXNAMELEN]; + char fsname[ZFS_MAX_DATASET_NAME_LEN]; error = dmu_fsname(nvpair_name(pair), fsname); if (error != 0) return (error); @@ -1378,9 +1393,9 @@ get_zfs_sb(const char *dsname, zfs_sb_t **zsbp) mutex_enter(&os->os_user_ptr_lock); *zsbp = dmu_objset_get_user(os); - if (*zsbp && (*zsbp)->z_sb) { - atomic_inc(&((*zsbp)->z_sb->s_active)); - } else { + /* bump s_active only when non-zero to prevent umount race */ + if (*zsbp == NULL || (*zsbp)->z_sb == NULL || + !atomic_inc_not_zero(&((*zsbp)->z_sb->s_active))) { error = SET_ERROR(ESRCH); } mutex_exit(&os->os_user_ptr_lock); @@ -2245,7 +2260,8 @@ zfs_ioc_snapshot_list_next(zfs_cmd_t *zc) * A dataset name of maximum length cannot have any snapshots, * so exit immediately. */ - if (strlcat(zc->zc_name, "@", sizeof (zc->zc_name)) >= MAXNAMELEN) { + if (strlcat(zc->zc_name, "@", sizeof (zc->zc_name)) >= + ZFS_MAX_DATASET_NAME_LEN) { dmu_objset_rele(os, FTAG); return (SET_ERROR(ESRCH)); } @@ -3033,7 +3049,7 @@ zfs_fill_zplprops(const char *dataset, nvlist_t *createprops, boolean_t fuids_ok, sa_ok; uint64_t zplver = ZPL_VERSION; objset_t *os = NULL; - char parentname[MAXNAMELEN]; + char parentname[ZFS_MAX_DATASET_NAME_LEN]; char *cp; spa_t *spa; uint64_t spa_vers; @@ -3182,8 +3198,25 @@ zfs_ioc_create(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl) if (error == 0) { error = zfs_set_prop_nvlist(fsname, ZPROP_SRC_LOCAL, nvprops, outnvl); - if (error != 0) - (void) dsl_destroy_head(fsname); + if (error != 0) { + spa_t *spa; + int error2; + + /* + * Volumes will return EBUSY and cannot be destroyed + * until all asynchronous minor handling has completed. + * Wait for the spa_zvol_taskq to drain then retry. + */ + error2 = dsl_destroy_head(fsname); + while ((error2 == EBUSY) && (type == DMU_OST_ZVOL)) { + error2 = spa_open(fsname, &spa, FTAG); + if (error2 == 0) { + taskq_wait(spa->spa_zvol_taskq); + spa_close(spa, FTAG); + } + error2 = dsl_destroy_head(fsname); + } + } } return (error); } @@ -3312,6 +3345,8 @@ zfs_ioc_log_history(const char *unused, nvlist_t *innvl, nvlist_t *outnvl) * we clear the TSD here. */ poolname = tsd_get(zfs_allow_log_key); + if (poolname == NULL) + return (SET_ERROR(EINVAL)); (void) tsd_set(zfs_allow_log_key, NULL); error = spa_open(poolname, &spa, FTAG); strfree(poolname); @@ -3382,7 +3417,7 @@ zfs_destroy_unmount_origin(const char *fsname) return; ds = dmu_objset_ds(os); if (dsl_dir_is_clone(ds->ds_dir) && DS_IS_DEFER_DESTROY(ds->ds_prev)) { - char originname[MAXNAMELEN]; + char originname[ZFS_MAX_DATASET_NAME_LEN]; dsl_dataset_name(ds->ds_prev, originname); dmu_objset_rele(os, FTAG); (void) zfs_unmount_snap(originname); @@ -3535,10 +3570,37 @@ zfs_ioc_destroy(zfs_cmd_t *zc) return (err); } - if (strchr(zc->zc_name, '@')) + if (strchr(zc->zc_name, '@')) { err = dsl_destroy_snapshot(zc->zc_name, zc->zc_defer_destroy); - else + } else { err = dsl_destroy_head(zc->zc_name); + if (err == EEXIST) { + /* + * It is possible that the given DS may have + * hidden child (%recv) datasets - "leftovers" + * resulting from the previously interrupted + * 'zfs receive'. + * + * 6 extra bytes for /%recv + */ + char namebuf[ZFS_MAX_DATASET_NAME_LEN + 6]; + + (void) snprintf(namebuf, sizeof (namebuf), + "%s/%s", zc->zc_name, recv_clone_name); + + /* + * Try to remove the hidden child (%recv) and after + * that try to remove the target dataset. + * If the hidden child (%recv) does not exist + * the original error (EEXIST) will be returned + */ + err = dsl_destroy_head(namebuf); + if (err == 0) + err = dsl_destroy_head(zc->zc_name); + else if (err == ENOENT) + err = EEXIST; + } + } return (err); } @@ -3761,7 +3823,7 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr) /* * If this is a bootable dataset then - * the we don't allow large (>128K) blocks, + * we don't allow large (>128K) blocks, * because GRUB doesn't support them. */ if (zfs_is_bootfs(dsname) && @@ -3789,6 +3851,34 @@ zfs_check_settable(const char *dsname, nvpair_t *pair, cred_t *cr) } break; + case ZFS_PROP_DNODESIZE: + /* Dnode sizes above 512 need the feature to be enabled */ + if (nvpair_value_uint64(pair, &intval) == 0 && + intval != ZFS_DNSIZE_LEGACY) { + spa_t *spa; + + /* + * If this is a bootable dataset then + * we don't allow large (>512B) dnodes, + * because GRUB doesn't support them. + */ + if (zfs_is_bootfs(dsname) && + intval != ZFS_DNSIZE_LEGACY) { + return (SET_ERROR(EDOM)); + } + + if ((err = spa_open(dsname, &spa, FTAG)) != 0) + return (err); + + if (!spa_feature_is_enabled(spa, + SPA_FEATURE_LARGE_DNODE)) { + spa_close(spa, FTAG); + return (SET_ERROR(ENOTSUP)); + } + spa_close(spa, FTAG); + } + break; + case ZFS_PROP_SHARESMB: if (zpl_earlier_version(dsname, ZPL_VERSION_FUID)) return (SET_ERROR(ENOTSUP)); @@ -3938,74 +4028,89 @@ props_reduce(nvlist_t *props, nvlist_t *origprops) } } +/* + * Extract properties that cannot be set PRIOR to the receipt of a dataset. + * For example, refquota cannot be set until after the receipt of a dataset, + * because in replication streams, an older/earlier snapshot may exceed the + * refquota. We want to receive the older/earlier snapshot, but setting + * refquota pre-receipt will set the dsl's ACTUAL quota, which will prevent + * the older/earlier snapshot from being received (with EDQUOT). + * + * The ZFS test "zfs_receive_011_pos" demonstrates such a scenario. + * + * libzfs will need to be judicious handling errors encountered by props + * extracted by this function. + */ +static nvlist_t * +extract_delay_props(nvlist_t *props) +{ + nvlist_t *delayprops; + nvpair_t *nvp, *tmp; + static const zfs_prop_t delayable[] = { ZFS_PROP_REFQUOTA, 0 }; + int i; + + VERIFY(nvlist_alloc(&delayprops, NV_UNIQUE_NAME, KM_SLEEP) == 0); + + for (nvp = nvlist_next_nvpair(props, NULL); nvp != NULL; + nvp = nvlist_next_nvpair(props, nvp)) { + /* + * strcmp() is safe because zfs_prop_to_name() always returns + * a bounded string. + */ + for (i = 0; delayable[i] != 0; i++) { + if (strcmp(zfs_prop_to_name(delayable[i]), + nvpair_name(nvp)) == 0) { + break; + } + } + if (delayable[i] != 0) { + tmp = nvlist_prev_nvpair(props, nvp); + VERIFY(nvlist_add_nvpair(delayprops, nvp) == 0); + VERIFY(nvlist_remove_nvpair(props, nvp) == 0); + nvp = tmp; + } + } + + if (nvlist_empty(delayprops)) { + nvlist_free(delayprops); + delayprops = NULL; + } + return (delayprops); +} + #ifdef DEBUG static boolean_t zfs_ioc_recv_inject_err; #endif /* - * inputs: - * zc_name name of containing filesystem - * zc_nvlist_src{_size} nvlist of properties to apply - * zc_value name of snapshot to create - * zc_string name of clone origin (if DRR_FLAG_CLONE) - * zc_cookie file descriptor to recv from - * zc_begin_record the BEGIN record of the stream (not byteswapped) - * zc_guid force flag - * zc_cleanup_fd cleanup-on-exit file descriptor - * zc_action_handle handle for this guid/ds mapping (or zero on first call) - * - * outputs: - * zc_cookie number of bytes read - * zc_nvlist_dst{_size} error for each unapplied received property - * zc_obj zprop_errflags_t - * zc_action_handle handle for this guid/ds mapping + * nvlist 'errors' is always allocated. It will contain descriptions of + * encountered errors, if any. It's the callers responsibility to free. */ static int -zfs_ioc_recv(zfs_cmd_t *zc) +zfs_ioc_recv_impl(char *tofs, char *tosnap, char *origin, + nvlist_t *props, boolean_t force, boolean_t resumable, int input_fd, + dmu_replay_record_t *begin_record, int cleanup_fd, uint64_t *read_bytes, + uint64_t *errflags, uint64_t *action_handle, nvlist_t **errors) { - file_t *fp; dmu_recv_cookie_t drc; - boolean_t force = (boolean_t)zc->zc_guid; - int fd; int error = 0; int props_error = 0; - nvlist_t *errors; offset_t off; - nvlist_t *props = NULL; /* sent properties */ + nvlist_t *delayprops = NULL; /* sent properties applied post-receive */ nvlist_t *origprops = NULL; /* existing properties */ - char *origin = NULL; - char *tosnap; - char tofs[ZFS_MAXNAMELEN]; boolean_t first_recvd_props = B_FALSE; + file_t *input_fp; - if (dataset_namecheck(zc->zc_value, NULL, NULL) != 0 || - strchr(zc->zc_value, '@') == NULL || - strchr(zc->zc_value, '%')) - return (SET_ERROR(EINVAL)); - - (void) strcpy(tofs, zc->zc_value); - tosnap = strchr(tofs, '@'); - *tosnap++ = '\0'; - - if (zc->zc_nvlist_src != 0 && - (error = get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size, - zc->zc_iflags, &props)) != 0) - return (error); + *read_bytes = 0; + *errflags = 0; + *errors = fnvlist_alloc(); - fd = zc->zc_cookie; - fp = getf(fd); - if (fp == NULL) { - nvlist_free(props); + input_fp = getf(input_fd); + if (input_fp == NULL) return (SET_ERROR(EBADF)); - } - - VERIFY(nvlist_alloc(&errors, NV_UNIQUE_NAME, KM_SLEEP) == 0); - - if (zc->zc_string[0]) - origin = zc->zc_string; error = dmu_recv_begin(tofs, tosnap, - &zc->zc_begin_record, force, origin, &drc); + begin_record, force, resumable, origin, &drc); if (error != 0) goto out; @@ -4038,14 +4143,14 @@ zfs_ioc_recv(zfs_cmd_t *zc) if (!first_recvd_props) props_reduce(props, origprops); if (zfs_check_clearable(tofs, origprops, &errlist) != 0) - (void) nvlist_merge(errors, errlist, 0); + (void) nvlist_merge(*errors, errlist, 0); nvlist_free(errlist); if (clear_received_props(tofs, origprops, first_recvd_props ? NULL : props) != 0) - zc->zc_obj |= ZPROP_ERR_NOCLEAR; + *errflags |= ZPROP_ERR_NOCLEAR; } else { - zc->zc_obj |= ZPROP_ERR_NOCLEAR; + *errflags |= ZPROP_ERR_NOCLEAR; } } @@ -4053,24 +4158,15 @@ zfs_ioc_recv(zfs_cmd_t *zc) props_error = dsl_prop_set_hasrecvd(tofs); if (props_error == 0) { + delayprops = extract_delay_props(props); (void) zfs_set_prop_nvlist(tofs, ZPROP_SRC_RECEIVED, - props, errors); + props, *errors); } } - if (zc->zc_nvlist_dst_size != 0 && - (nvlist_smush(errors, zc->zc_nvlist_dst_size) != 0 || - put_nvlist(zc, errors) != 0)) { - /* - * Caller made zc->zc_nvlist_dst less than the minimum expected - * size or supplied an invalid address. - */ - props_error = SET_ERROR(EINVAL); - } - - off = fp->f_offset; - error = dmu_recv_stream(&drc, fp->f_vnode, &off, zc->zc_cleanup_fd, - &zc->zc_action_handle); + off = input_fp->f_offset; + error = dmu_recv_stream(&drc, input_fp->f_vnode, &off, cleanup_fd, + action_handle); if (error == 0) { zfs_sb_t *zsb = NULL; @@ -4092,11 +4188,32 @@ zfs_ioc_recv(zfs_cmd_t *zc) } else { error = dmu_recv_end(&drc, NULL); } + + /* Set delayed properties now, after we're done receiving. */ + if (delayprops != NULL && error == 0) { + (void) zfs_set_prop_nvlist(tofs, ZPROP_SRC_RECEIVED, + delayprops, *errors); + } + } + + if (delayprops != NULL) { + /* + * Merge delayed props back in with initial props, in case + * we're DEBUG and zfs_ioc_recv_inject_err is set (which means + * we have to make sure clear_received_props() includes + * the delayed properties). + * + * Since zfs_ioc_recv_inject_err is only in DEBUG kernels, + * using ASSERT() will be just like a VERIFY. + */ + ASSERT(nvlist_merge(props, delayprops, 0) == 0); + nvlist_free(delayprops); } - zc->zc_cookie = off - fp->f_offset; - if (VOP_SEEK(fp->f_vnode, fp->f_offset, &off, NULL) == 0) - fp->f_offset = off; + + *read_bytes = off - input_fp->f_offset; + if (VOP_SEEK(input_fp->f_vnode, input_fp->f_offset, &off, NULL) == 0) + input_fp->f_offset = off; #ifdef DEBUG if (zfs_ioc_recv_inject_err) { @@ -4115,14 +4232,14 @@ zfs_ioc_recv(zfs_cmd_t *zc) * Since we may have left a $recvd value on the * system, we can't clear the $hasrecvd flag. */ - zc->zc_obj |= ZPROP_ERR_NORESTORE; + *errflags |= ZPROP_ERR_NORESTORE; } else if (first_recvd_props) { dsl_prop_unset_hasrecvd(tofs); } if (origprops == NULL && !drc.drc_newfs) { /* We failed to stash the original properties. */ - zc->zc_obj |= ZPROP_ERR_NORESTORE; + *errflags |= ZPROP_ERR_NORESTORE; } /* @@ -4139,14 +4256,12 @@ zfs_ioc_recv(zfs_cmd_t *zc) * We stashed the original properties but failed to * restore them. */ - zc->zc_obj |= ZPROP_ERR_NORESTORE; + *errflags |= ZPROP_ERR_NORESTORE; } } out: - nvlist_free(props); + releasef(input_fd); nvlist_free(origprops); - nvlist_free(errors); - releasef(fd); if (error == 0) error = props_error; @@ -4154,6 +4269,176 @@ zfs_ioc_recv(zfs_cmd_t *zc) return (error); } +/* + * inputs: + * zc_name name of containing filesystem (unused) + * zc_nvlist_src{_size} nvlist of properties to apply + * zc_value name of snapshot to create + * zc_string name of clone origin (if DRR_FLAG_CLONE) + * zc_cookie file descriptor to recv from + * zc_begin_record the BEGIN record of the stream (not byteswapped) + * zc_guid force flag + * zc_cleanup_fd cleanup-on-exit file descriptor + * zc_action_handle handle for this guid/ds mapping (or zero on first call) + * + * outputs: + * zc_cookie number of bytes read + * zc_obj zprop_errflags_t + * zc_action_handle handle for this guid/ds mapping + * zc_nvlist_dst{_size} error for each unapplied received property + */ +static int +zfs_ioc_recv(zfs_cmd_t *zc) +{ + dmu_replay_record_t begin_record; + nvlist_t *errors = NULL; + nvlist_t *props = NULL; + char *origin = NULL; + char *tosnap; + char tofs[ZFS_MAX_DATASET_NAME_LEN]; + int error = 0; + + if (dataset_namecheck(zc->zc_value, NULL, NULL) != 0 || + strchr(zc->zc_value, '@') == NULL || + strchr(zc->zc_value, '%')) + return (SET_ERROR(EINVAL)); + + (void) strcpy(tofs, zc->zc_value); + tosnap = strchr(tofs, '@'); + *tosnap++ = '\0'; + + if (zc->zc_nvlist_src != 0 && + (error = get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size, + zc->zc_iflags, &props)) != 0) + return (error); + + if (zc->zc_string[0]) + origin = zc->zc_string; + + begin_record.drr_type = DRR_BEGIN; + begin_record.drr_payloadlen = 0; + begin_record.drr_u.drr_begin = zc->zc_begin_record; + + error = zfs_ioc_recv_impl(tofs, tosnap, origin, props, zc->zc_guid, + B_FALSE, zc->zc_cookie, &begin_record, zc->zc_cleanup_fd, + &zc->zc_cookie, &zc->zc_obj, &zc->zc_action_handle, &errors); + nvlist_free(props); + + /* + * Now that all props, initial and delayed, are set, report the prop + * errors to the caller. + */ + if (zc->zc_nvlist_dst_size != 0 && errors != NULL && + (nvlist_smush(errors, zc->zc_nvlist_dst_size) != 0 || + put_nvlist(zc, errors) != 0)) { + /* + * Caller made zc->zc_nvlist_dst less than the minimum expected + * size or supplied an invalid address. + */ + error = SET_ERROR(EINVAL); + } + + nvlist_free(errors); + + return (error); +} + +/* + * innvl: { + * "snapname" -> full name of the snapshot to create + * (optional) "props" -> properties to set (nvlist) + * (optional) "origin" -> name of clone origin (DRR_FLAG_CLONE) + * "begin_record" -> non-byteswapped dmu_replay_record_t + * "input_fd" -> file descriptor to read stream from (int32) + * (optional) "force" -> force flag (value ignored) + * (optional) "resumable" -> resumable flag (value ignored) + * (optional) "cleanup_fd" -> cleanup-on-exit file descriptor + * (optional) "action_handle" -> handle for this guid/ds mapping + * } + * + * outnvl: { + * "read_bytes" -> number of bytes read + * "error_flags" -> zprop_errflags_t + * "action_handle" -> handle for this guid/ds mapping + * "errors" -> error for each unapplied received property (nvlist) + * } + */ +static int +zfs_ioc_recv_new(const char *fsname, nvlist_t *innvl, nvlist_t *outnvl) +{ + dmu_replay_record_t *begin_record; + uint_t begin_record_size; + nvlist_t *errors = NULL; + nvlist_t *props = NULL; + char *snapname = NULL; + char *origin = NULL; + char *tosnap; + char tofs[ZFS_MAX_DATASET_NAME_LEN]; + boolean_t force; + boolean_t resumable; + uint64_t action_handle = 0; + uint64_t read_bytes = 0; + uint64_t errflags = 0; + int input_fd = -1; + int cleanup_fd = -1; + int error; + + error = nvlist_lookup_string(innvl, "snapname", &snapname); + if (error != 0) + return (SET_ERROR(EINVAL)); + + if (dataset_namecheck(snapname, NULL, NULL) != 0 || + strchr(snapname, '@') == NULL || + strchr(snapname, '%')) + return (SET_ERROR(EINVAL)); + + (void) strcpy(tofs, snapname); + tosnap = strchr(tofs, '@'); + *tosnap++ = '\0'; + + error = nvlist_lookup_string(innvl, "origin", &origin); + if (error && error != ENOENT) + return (error); + + error = nvlist_lookup_byte_array(innvl, "begin_record", + (uchar_t **) &begin_record, &begin_record_size); + if (error != 0 || begin_record_size != sizeof (*begin_record)) + return (SET_ERROR(EINVAL)); + + error = nvlist_lookup_int32(innvl, "input_fd", &input_fd); + if (error != 0) + return (SET_ERROR(EINVAL)); + + force = nvlist_exists(innvl, "force"); + resumable = nvlist_exists(innvl, "resumable"); + + error = nvlist_lookup_int32(innvl, "cleanup_fd", &cleanup_fd); + if (error && error != ENOENT) + return (error); + + error = nvlist_lookup_uint64(innvl, "action_handle", &action_handle); + if (error && error != ENOENT) + return (error); + + error = nvlist_lookup_nvlist(innvl, "props", &props); + if (error && error != ENOENT) + return (error); + + error = zfs_ioc_recv_impl(tofs, tosnap, origin, props, force, + resumable, input_fd, begin_record, cleanup_fd, &read_bytes, + &errflags, &action_handle, &errors); + + fnvlist_add_uint64(outnvl, "read_bytes", read_bytes); + fnvlist_add_uint64(outnvl, "error_flags", errflags); + fnvlist_add_uint64(outnvl, "action_handle", action_handle); + fnvlist_add_nvlist(outnvl, "errors", errors); + + nvlist_free(errors); + nvlist_free(props); + + return (error); +} + /* * inputs: * zc_name name of snapshot to send @@ -5130,6 +5415,8 @@ zfs_ioc_space_snaps(const char *lastsnap, nvlist_t *innvl, nvlist_t *outnvl) * indicates that blocks > 128KB are permitted * (optional) "embedok" -> (value ignored) * presence indicates DRR_WRITE_EMBEDDED records are permitted + * (optional) "resume_object" and "resume_offset" -> (uint64) + * if present, resume send stream from specified object and offset. * } * * outnvl is unused @@ -5145,6 +5432,8 @@ zfs_ioc_send_new(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl) file_t *fp; boolean_t largeblockok; boolean_t embedok; + uint64_t resumeobj = 0; + uint64_t resumeoff = 0; error = nvlist_lookup_int32(innvl, "fd", &fd); if (error != 0) @@ -5155,12 +5444,15 @@ zfs_ioc_send_new(const char *snapname, nvlist_t *innvl, nvlist_t *outnvl) largeblockok = nvlist_exists(innvl, "largeblockok"); embedok = nvlist_exists(innvl, "embedok"); + (void) nvlist_lookup_uint64(innvl, "resume_object", &resumeobj); + (void) nvlist_lookup_uint64(innvl, "resume_offset", &resumeoff); + if ((fp = getf(fd)) == NULL) return (SET_ERROR(EBADF)); off = fp->f_offset; - error = dmu_send(snapname, fromname, embedok, largeblockok, - fd, fp->f_vnode, &off); + error = dmu_send(snapname, fromname, embedok, largeblockok, fd, + resumeobj, resumeoff, fp->f_vnode, &off); if (VOP_SEEK(fp->f_vnode, fp->f_offset, &off, NULL) == 0) fp->f_offset = off; @@ -5418,6 +5710,10 @@ zfs_ioctl_init(void) POOL_NAME, POOL_CHECK_SUSPENDED | POOL_CHECK_READONLY, B_TRUE, B_TRUE); + zfs_ioctl_register("receive", ZFS_IOC_RECV_NEW, + zfs_ioc_recv_new, zfs_secpolicy_recv_new, DATASET_NAME, + POOL_CHECK_SUSPENDED | POOL_CHECK_READONLY, B_TRUE, B_TRUE); + /* IOCTLS that use the legacy function signature */ zfs_ioctl_register_legacy(ZFS_IOC_POOL_FREEZE, zfs_ioc_pool_freeze, @@ -5795,7 +6091,23 @@ zfsdev_ioctl(struct file *filp, unsigned cmd, unsigned long arg) } zc->zc_iflags = flag & FKIOCTL; - if (zc->zc_nvlist_src_size != 0) { + if (zc->zc_nvlist_src_size > MAX_NVLIST_SRC_SIZE) { + /* + * Make sure the user doesn't pass in an insane value for + * zc_nvlist_src_size. We have to check, since we will end + * up allocating that much memory inside of get_nvlist(). This + * prevents a nefarious user from allocating tons of kernel + * memory. + * + * Also, we return EINVAL instead of ENOMEM here. The reason + * being that returning ENOMEM from an ioctl() has a special + * connotation; that the user's size value is too small and + * needs to be expanded to hold the nvlist. See + * zcmd_expand_dst_nvlist() for details. + */ + error = SET_ERROR(EINVAL); /* User's size too big */ + + } else if (zc->zc_nvlist_src_size != 0) { error = get_nvlist(zc->zc_nvlist_src, zc->zc_nvlist_src_size, zc->zc_iflags, &innvl); if (error != 0) @@ -5987,7 +6299,9 @@ static void zfs_allow_log_destroy(void *arg) { char *poolname = arg; - strfree(poolname); + + if (poolname != NULL) + strfree(poolname); } #ifdef DEBUG diff --git a/module/zfs/zfs_log.c b/module/zfs/zfs_log.c index 38d8de0ebf97..69efb3c16132 100644 --- a/module/zfs/zfs_log.c +++ b/module/zfs/zfs_log.c @@ -279,14 +279,16 @@ zfs_log_create(zilog_t *zilog, dmu_tx_t *tx, uint64_t txtype, lr = (lr_create_t *)&itx->itx_lr; lr->lr_doid = dzp->z_id; lr->lr_foid = zp->z_id; + /* Store dnode slot count in 8 bits above object id. */ + LR_FOID_SET_SLOTS(lr->lr_foid, zp->z_dnodesize >> DNODE_SHIFT); lr->lr_mode = zp->z_mode; - if (!IS_EPHEMERAL(zp->z_uid)) { - lr->lr_uid = (uint64_t)zp->z_uid; + if (!IS_EPHEMERAL(KUID_TO_SUID(ZTOI(zp)->i_uid))) { + lr->lr_uid = (uint64_t)KUID_TO_SUID(ZTOI(zp)->i_uid); } else { lr->lr_uid = fuidp->z_fuid_owner; } - if (!IS_EPHEMERAL(zp->z_gid)) { - lr->lr_gid = (uint64_t)zp->z_gid; + if (!IS_EPHEMERAL(KGID_TO_SGID(ZTOI(zp)->i_gid))) { + lr->lr_gid = (uint64_t)KGID_TO_SGID(ZTOI(zp)->i_gid); } else { lr->lr_gid = fuidp->z_fuid_group; } @@ -405,8 +407,8 @@ zfs_log_symlink(zilog_t *zilog, dmu_tx_t *tx, uint64_t txtype, lr = (lr_create_t *)&itx->itx_lr; lr->lr_doid = dzp->z_id; lr->lr_foid = zp->z_id; - lr->lr_uid = zp->z_uid; - lr->lr_gid = zp->z_gid; + lr->lr_uid = KUID_TO_SUID(ZTOI(zp)->i_uid); + lr->lr_gid = KGID_TO_SGID(ZTOI(zp)->i_gid); lr->lr_mode = zp->z_mode; (void) sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(ZTOZSB(zp)), &lr->lr_gen, sizeof (uint64_t)); diff --git a/module/zfs/zfs_replay.c b/module/zfs/zfs_replay.c index b97a60ed8a1d..54c17543727c 100644 --- a/module/zfs/zfs_replay.c +++ b/module/zfs/zfs_replay.c @@ -279,6 +279,8 @@ zfs_replay_create_acl(zfs_sb_t *zsb, lr_acl_create_t *lracl, boolean_t byteswap) void *fuidstart; size_t xvatlen = 0; uint64_t txtype; + uint64_t objid; + uint64_t dnodesize; int error; txtype = (lr->lr_common.lrc_txtype & ~TX_CI); @@ -304,19 +306,24 @@ zfs_replay_create_acl(zfs_sb_t *zsb, lr_acl_create_t *lracl, boolean_t byteswap) if ((error = zfs_zget(zsb, lr->lr_doid, &dzp)) != 0) return (error); + objid = LR_FOID_GET_OBJ(lr->lr_foid); + dnodesize = LR_FOID_GET_SLOTS(lr->lr_foid) << DNODE_SHIFT; + xva_init(&xva); zfs_init_vattr(&xva.xva_vattr, ATTR_MODE | ATTR_UID | ATTR_GID, - lr->lr_mode, lr->lr_uid, lr->lr_gid, lr->lr_rdev, lr->lr_foid); + lr->lr_mode, lr->lr_uid, lr->lr_gid, lr->lr_rdev, objid); /* * All forms of zfs create (create, mkdir, mkxattrdir, symlink) * eventually end up in zfs_mknode(), which assigns the object's - * creation time and generation number. The generic zfs_create() - * doesn't have either concept, so we smuggle the values inside - * the vattr's otherwise unused va_ctime and va_nblocks fields. + * creation time, generation number, and dnode size. The generic + * zfs_create() has no concept of these attributes, so we smuggle + * the values inside the vattr's otherwise unused va_ctime, + * va_nblocks, and va_fsid fields. */ ZFS_TIME_DECODE(&xva.xva_vattr.va_ctime, lr->lr_crtime); xva.xva_vattr.va_nblocks = lr->lr_gen; + xva.xva_vattr.va_fsid = dnodesize; error = dmu_object_info(zsb->z_os, lr->lr_foid, NULL); if (error != ENOENT) @@ -418,6 +425,8 @@ zfs_replay_create(zfs_sb_t *zsb, lr_create_t *lr, boolean_t byteswap) void *start; size_t xvatlen; uint64_t txtype; + uint64_t objid; + uint64_t dnodesize; int error; txtype = (lr->lr_common.lrc_txtype & ~TX_CI); @@ -431,21 +440,26 @@ zfs_replay_create(zfs_sb_t *zsb, lr_create_t *lr, boolean_t byteswap) if ((error = zfs_zget(zsb, lr->lr_doid, &dzp)) != 0) return (error); + objid = LR_FOID_GET_OBJ(lr->lr_foid); + dnodesize = LR_FOID_GET_SLOTS(lr->lr_foid) << DNODE_SHIFT; + xva_init(&xva); zfs_init_vattr(&xva.xva_vattr, ATTR_MODE | ATTR_UID | ATTR_GID, - lr->lr_mode, lr->lr_uid, lr->lr_gid, lr->lr_rdev, lr->lr_foid); + lr->lr_mode, lr->lr_uid, lr->lr_gid, lr->lr_rdev, objid); /* * All forms of zfs create (create, mkdir, mkxattrdir, symlink) * eventually end up in zfs_mknode(), which assigns the object's - * creation time and generation number. The generic zfs_create() - * doesn't have either concept, so we smuggle the values inside - * the vattr's otherwise unused va_ctime and va_nblocks fields. + * creation time, generation number, and dnode slot count. The + * generic zfs_create() has no concept of these attributes, so + * we smuggle the values inside * the vattr's otherwise unused + * va_ctime, va_nblocks, and va_nlink fields. */ ZFS_TIME_DECODE(&xva.xva_vattr.va_ctime, lr->lr_crtime); xva.xva_vattr.va_nblocks = lr->lr_gen; + xva.xva_vattr.va_fsid = dnodesize; - error = dmu_object_info(zsb->z_os, lr->lr_foid, NULL); + error = dmu_object_info(zsb->z_os, objid, NULL); if (error != ENOENT) goto out; diff --git a/module/zfs/zfs_sa.c b/module/zfs/zfs_sa.c index f4841435b562..33b767808cde 100644 --- a/module/zfs/zfs_sa.c +++ b/module/zfs/zfs_sa.c @@ -97,8 +97,7 @@ zfs_sa_symlink(znode_t *zp, char *link, int len, dmu_tx_t *tx) dmu_buf_t *db = sa_get_db(zp->z_sa_hdl); if (ZFS_OLD_ZNODE_PHYS_SIZE + len <= dmu_bonus_max()) { - VERIFY(dmu_set_bonus(db, - len + ZFS_OLD_ZNODE_PHYS_SIZE, tx) == 0); + VERIFY0(dmu_set_bonus(db, len + ZFS_OLD_ZNODE_PHYS_SIZE, tx)); if (len) { bcopy(link, (caddr_t)db->db_data + ZFS_OLD_ZNODE_PHYS_SIZE, len); @@ -107,8 +106,8 @@ zfs_sa_symlink(znode_t *zp, char *link, int len, dmu_tx_t *tx) dmu_buf_t *dbp; zfs_grow_blocksize(zp, len, tx); - VERIFY(0 == dmu_buf_hold(ZTOZSB(zp)->z_os, - zp->z_id, 0, FTAG, &dbp, DMU_READ_NO_PREFETCH)); + VERIFY0(dmu_buf_hold(ZTOZSB(zp)->z_os, zp->z_id, 0, FTAG, &dbp, + DMU_READ_NO_PREFETCH)); dmu_buf_will_dirty(dbp, tx); @@ -278,6 +277,7 @@ zfs_sa_upgrade(sa_handle_t *hdl, dmu_tx_t *tx) zfs_acl_locator_cb_t locate = { 0 }; uint64_t uid, gid, mode, rdev, xattr, parent, tmp_gen; uint64_t crtime[2], mtime[2], ctime[2], atime[2]; + uint64_t links; zfs_acl_phys_t znode_acl; char scanstamp[AV_SCANSTAMP_SZ]; boolean_t drop_lock = B_FALSE; @@ -352,8 +352,9 @@ zfs_sa_upgrade(sa_handle_t *hdl, dmu_tx_t *tx) &ctime, 16); SA_ADD_BULK_ATTR(sa_attrs, count, SA_ZPL_CRTIME(zsb), NULL, &crtime, 16); + links = ZTOI(zp)->i_nlink; SA_ADD_BULK_ATTR(sa_attrs, count, SA_ZPL_LINKS(zsb), NULL, - &zp->z_links, 8); + &links, 8); if (S_ISBLK(ZTOI(zp)->i_mode) || S_ISCHR(ZTOI(zp)->i_mode)) SA_ADD_BULK_ATTR(sa_attrs, count, SA_ZPL_RDEV(zsb), NULL, &rdev, 8); diff --git a/module/zfs/zfs_vfsops.c b/module/zfs/zfs_vfsops.c index 7696071f1fe7..d8b27461aacc 100644 --- a/module/zfs/zfs_vfsops.c +++ b/module/zfs/zfs_vfsops.c @@ -627,10 +627,11 @@ zfs_owner_overquota(zfs_sb_t *zsb, znode_t *zp, boolean_t isgroup) { uint64_t fuid; uint64_t quotaobj; + struct inode *ip = ZTOI(zp); quotaobj = isgroup ? zsb->z_groupquota_obj : zsb->z_userquota_obj; - fuid = isgroup ? zp->z_gid : zp->z_uid; + fuid = isgroup ? KGID_TO_SGID(ip->i_gid) : KUID_TO_SUID(ip->i_uid); if (quotaobj == 0 || zsb->z_replay) return (B_FALSE); @@ -1020,7 +1021,7 @@ zfs_statvfs(struct dentry *dentry, struct kstatfs *statp) statp->f_fsid.val[0] = (uint32_t)fsid; statp->f_fsid.val[1] = (uint32_t)(fsid >> 32); statp->f_type = ZFS_SUPER_MAGIC; - statp->f_namelen = ZFS_MAXNAMELEN; + statp->f_namelen = MAXNAMELEN - 1; /* * We have all of 40 characters to stuff a string here. @@ -1050,8 +1051,7 @@ zfs_root(zfs_sb_t *zsb, struct inode **ipp) } EXPORT_SYMBOL(zfs_root); -#if !defined(HAVE_SPLIT_SHRINKER_CALLBACK) && !defined(HAVE_SHRINK) && \ - defined(HAVE_D_PRUNE_ALIASES) +#ifdef HAVE_D_PRUNE_ALIASES /* * Linux kernels older than 3.1 do not support a per-filesystem shrinker. * To accommodate this we must improvise and manually walk the list of znodes @@ -1141,15 +1141,29 @@ zfs_sb_prune(struct super_block *sb, unsigned long nr_to_scan, int *objects) } else { *objects = (*shrinker->scan_objects)(shrinker, &sc); } + #elif defined(HAVE_SPLIT_SHRINKER_CALLBACK) *objects = (*shrinker->scan_objects)(shrinker, &sc); #elif defined(HAVE_SHRINK) *objects = (*shrinker->shrink)(shrinker, &sc); #elif defined(HAVE_D_PRUNE_ALIASES) +#define D_PRUNE_ALIASES_IS_DEFAULT *objects = zfs_sb_prune_aliases(zsb, nr_to_scan); #else #error "No available dentry and inode cache pruning mechanism." #endif + +#if defined(HAVE_D_PRUNE_ALIASES) && !defined(D_PRUNE_ALIASES_IS_DEFAULT) +#undef D_PRUNE_ALIASES_IS_DEFAULT + /* + * Fall back to zfs_sb_prune_aliases if the kernel's per-superblock + * shrinker couldn't free anything, possibly due to the inodes being + * allocated in a different memcg. + */ + if (*objects == 0) + *objects = zfs_sb_prune_aliases(zsb, nr_to_scan); +#endif + ZFS_EXIT(zsb); dprintf_ds(zsb->z_os->os_dsl_dataset, @@ -1356,7 +1370,8 @@ zfs_domount(struct super_block *sb, zfs_mntopts_t *zmo, int silent) dmu_objset_set_user(zsb->z_os, zsb); mutex_exit(&zsb->z_os->os_user_ptr_lock); } else { - error = zfs_sb_setup(zsb, B_TRUE); + if ((error = zfs_sb_setup(zsb, B_TRUE))) + goto out; } /* Allocate a root inode for the filesystem. */ @@ -1382,6 +1397,11 @@ zfs_domount(struct super_block *sb, zfs_mntopts_t *zmo, int silent) if (error) { dmu_objset_disown(zsb->z_os, zsb); zfs_sb_free(zsb); + /* + * make sure we don't have dangling sb->s_fs_info which + * zfs_preumount will use. + */ + sb->s_fs_info = NULL; } return (error); @@ -1400,8 +1420,29 @@ zfs_preumount(struct super_block *sb) { zfs_sb_t *zsb = sb->s_fs_info; - if (zsb) + /* zsb is NULL when zfs_domount fails during mount */ + if (zsb) { zfsctl_destroy(sb->s_fs_info); + /* + * Wait for iput_async before entering evict_inodes in + * generic_shutdown_super. The reason we must finish before + * evict_inodes is when lazytime is on, or when zfs_purgedir + * calls zfs_zget, iput would bump i_count from 0 to 1. This + * would race with the i_count check in evict_inodes. This means + * it could destroy the inode while we are still using it. + * + * We wait for two passes. xattr directories in the first pass + * may add xattr entries in zfs_purgedir, so in the second pass + * we wait for them. We don't use taskq_wait here because it is + * a pool wide taskq. Other mounted filesystems can constantly + * do iput_async and there's no guarantee when taskq will be + * empty. + */ + taskq_wait_outstanding(dsl_pool_iput_taskq( + dmu_objset_pool(zsb->z_os)), 0); + taskq_wait_outstanding(dsl_pool_iput_taskq( + dmu_objset_pool(zsb->z_os)), 0); + } } EXPORT_SYMBOL(zfs_preumount); @@ -1527,6 +1568,14 @@ zfs_vget(struct super_block *sb, struct inode **ipp, fid_t *fidp) ZFS_EXIT(zsb); return (err); } + + /* Don't export xattr stuff */ + if (zp->z_pflags & ZFS_XATTR) { + iput(ZTOI(zp)); + ZFS_EXIT(zsb); + return (SET_ERROR(ENOENT)); + } + (void) sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zsb), &zp_gen, sizeof (uint64_t)); zp_gen = zp_gen & gen_mask; @@ -1539,7 +1588,7 @@ zfs_vget(struct super_block *sb, struct inode **ipp, fid_t *fidp) fid_gen); iput(ZTOI(zp)); ZFS_EXIT(zsb); - return (SET_ERROR(EINVAL)); + return (SET_ERROR(ENOENT)); } *ipp = ZTOI(zp); diff --git a/module/zfs/zfs_vnops.c b/module/zfs/zfs_vnops.c index e53992cbbd88..bde2140d013d 100644 --- a/module/zfs/zfs_vnops.c +++ b/module/zfs/zfs_vnops.c @@ -602,6 +602,7 @@ zfs_write(struct inode *ip, uio_t *uio, int ioflag, cred_t *cr) int count = 0; sa_bulk_attr_t bulk[4]; uint64_t mtime[2], ctime[2]; + uint32_t uid; ASSERTV(int iovcnt = uio->uio_iovcnt); /* @@ -862,11 +863,12 @@ zfs_write(struct inode *ip, uio_t *uio, int ioflag, cred_t *cr) * user 0 is not an ephemeral uid. */ mutex_enter(&zp->z_acl_lock); + uid = KUID_TO_SUID(ip->i_uid); if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) | (S_IXUSR >> 6))) != 0 && (zp->z_mode & (S_ISUID | S_ISGID)) != 0 && secpolicy_vnode_setid_retain(cr, - (zp->z_mode & S_ISUID) != 0 && zp->z_uid == 0) != 0) { + ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) { uint64_t newmode; zp->z_mode &= ~(S_ISUID | S_ISGID); newmode = zp->z_mode; @@ -1524,6 +1526,7 @@ zfs_remove(struct inode *dip, char *name, cred_t *cr, int flags) uint64_t acl_obj, xattr_obj; uint64_t xattr_obj_unlinked = 0; uint64_t obj = 0; + uint64_t links; zfs_dirlock_t *dl; dmu_tx_t *tx; boolean_t may_delete_now, delete_now = FALSE; @@ -1672,12 +1675,13 @@ zfs_remove(struct inode *dip, char *name, cred_t *cr, int flags) if (delete_now) { if (xattr_obj_unlinked) { - ASSERT3U(xzp->z_links, ==, 2); + ASSERT3U(ZTOI(xzp)->i_nlink, ==, 2); mutex_enter(&xzp->z_lock); xzp->z_unlinked = 1; - xzp->z_links = 0; + clear_nlink(ZTOI(xzp)); + links = 0; error = sa_update(xzp->z_sa_hdl, SA_ZPL_LINKS(zsb), - &xzp->z_links, sizeof (xzp->z_links), tx); + &links, sizeof (links), tx); ASSERT3U(error, ==, 0); mutex_exit(&xzp->z_lock); zfs_unlinked_add(xzp, tx); @@ -2297,9 +2301,9 @@ zfs_getattr(struct inode *ip, vattr_t *vap, int flags, cred_t *cr) vap->va_fsid = ZTOI(zp)->i_sb->s_dev; vap->va_nodeid = zp->z_id; if ((zp->z_id == zsb->z_root) && zfs_show_ctldir(zp)) - links = zp->z_links + 1; + links = ZTOI(zp)->i_nlink + 1; else - links = zp->z_links; + links = ZTOI(zp)->i_nlink; vap->va_nlink = MIN(links, ZFS_LINK_MAX); vap->va_size = i_size_read(ip); vap->va_rdev = ip->i_rdev; @@ -2842,7 +2846,7 @@ zfs_setattr(struct inode *ip, vattr_t *vap, int flags, cred_t *cr) if (mask & ATTR_UID) { new_uid = zfs_fuid_create(zsb, (uint64_t)vap->va_uid, cr, ZFS_OWNER, &fuidp); - if (new_uid != zp->z_uid && + if (new_uid != KUID_TO_SUID(ZTOI(zp)->i_uid) && zfs_fuid_overquota(zsb, B_FALSE, new_uid)) { if (attrzp) iput(ZTOI(attrzp)); @@ -2854,7 +2858,7 @@ zfs_setattr(struct inode *ip, vattr_t *vap, int flags, cred_t *cr) if (mask & ATTR_GID) { new_gid = zfs_fuid_create(zsb, (uint64_t)vap->va_gid, cr, ZFS_GROUP, &fuidp); - if (new_gid != zp->z_gid && + if (new_gid != KGID_TO_SGID(ZTOI(zp)->i_gid) && zfs_fuid_overquota(zsb, B_TRUE, new_gid)) { if (attrzp) iput(ZTOI(attrzp)); @@ -2948,24 +2952,24 @@ zfs_setattr(struct inode *ip, vattr_t *vap, int flags, cred_t *cr) if (mask & ATTR_UID) { SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zsb), NULL, &new_uid, sizeof (new_uid)); - zp->z_uid = new_uid; + ZTOI(zp)->i_uid = SUID_TO_KUID(new_uid); if (attrzp) { SA_ADD_BULK_ATTR(xattr_bulk, xattr_count, SA_ZPL_UID(zsb), NULL, &new_uid, sizeof (new_uid)); - attrzp->z_uid = new_uid; + ZTOI(attrzp)->i_uid = SUID_TO_KUID(new_uid); } } if (mask & ATTR_GID) { SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zsb), NULL, &new_gid, sizeof (new_gid)); - zp->z_gid = new_gid; + ZTOI(zp)->i_gid = SGID_TO_KGID(new_gid); if (attrzp) { SA_ADD_BULK_ATTR(xattr_bulk, xattr_count, SA_ZPL_GID(zsb), NULL, &new_gid, sizeof (new_gid)); - attrzp->z_gid = new_gid; + ZTOI(attrzp)->i_gid = SGID_TO_KGID(new_gid); } } if (!(mask & ATTR_MODE)) { @@ -3845,7 +3849,7 @@ zfs_link(struct inode *tdip, struct inode *sip, char *name, cred_t *cr, return (SET_ERROR(EINVAL)); } - owner = zfs_fuid_map_id(zsb, szp->z_uid, cr, ZFS_OWNER); + owner = zfs_fuid_map_id(zsb, KUID_TO_SUID(sip->i_uid), cr, ZFS_OWNER); if (owner != crgetuid(cr) && secpolicy_basic_link(cr) != 0) { ZFS_EXIT(zsb); return (SET_ERROR(EPERM)); @@ -4271,10 +4275,10 @@ zfs_fillpage(struct inode *ip, struct page *pl[], int nr_pages) * Iterate over list of pages and read each page individually. */ page_idx = 0; - cur_pp = pl[0]; for (total = io_off + io_len; io_off < total; io_off += PAGESIZE) { caddr_t va; + cur_pp = pl[page_idx++]; va = kmap(cur_pp); err = dmu_read(os, zp->z_id, io_off, PAGESIZE, va, DMU_READ_PREFETCH); @@ -4285,7 +4289,6 @@ zfs_fillpage(struct inode *ip, struct page *pl[], int nr_pages) err = SET_ERROR(EIO); return (err); } - cur_pp = pl[++page_idx]; } return (0); diff --git a/module/zfs/zfs_znode.c b/module/zfs/zfs_znode.c index d9fcd2912919..74502935c4e9 100644 --- a/module/zfs/zfs_znode.c +++ b/module/zfs/zfs_znode.c @@ -62,6 +62,7 @@ #include #include +#include #include #include #include @@ -118,7 +119,6 @@ zfs_znode_cache_constructor(void *buf, void *arg, int kmflags) zp->z_dirlocks = NULL; zp->z_acl_cached = NULL; zp->z_xattr_cached = NULL; - zp->z_xattr_parent = NULL; zp->z_moved = 0; return (0); } @@ -140,7 +140,6 @@ zfs_znode_cache_destructor(void *buf, void *arg) ASSERT(zp->z_dirlocks == NULL); ASSERT(zp->z_acl_cached == NULL); ASSERT(zp->z_xattr_cached == NULL); - ASSERT(zp->z_xattr_parent == NULL); } static int @@ -432,11 +431,6 @@ zfs_inode_destroy(struct inode *ip) zp->z_xattr_cached = NULL; } - if (zp->z_xattr_parent) { - zfs_iput_async(ZTOI(zp->z_xattr_parent)); - zp->z_xattr_parent = NULL; - } - kmem_cache_free(znode_cache, zp); } @@ -539,12 +533,8 @@ zfs_inode_update_impl(znode_t *zp, boolean_t new) dmu_object_size_from_db(sa_get_db(zp->z_sa_hdl), &blksize, &i_blocks); spin_lock(&ip->i_lock); - ip->i_uid = SUID_TO_KUID(zp->z_uid); - ip->i_gid = SGID_TO_KGID(zp->z_gid); - set_nlink(ip, zp->z_links); ip->i_mode = zp->z_mode; zfs_set_inode_flags(zp, ip); - ip->i_blkbits = SPA_MINBLOCKSHIFT; ip->i_blocks = i_blocks; /* @@ -581,14 +571,15 @@ zfs_inode_update(znode_t *zp) */ static znode_t * zfs_znode_alloc(zfs_sb_t *zsb, dmu_buf_t *db, int blksz, - dmu_object_type_t obj_type, uint64_t obj, sa_handle_t *hdl, - struct inode *dip) + dmu_object_type_t obj_type, uint64_t obj, sa_handle_t *hdl) { znode_t *zp; struct inode *ip; uint64_t mode; uint64_t parent; uint64_t tmp_gen; + uint64_t links; + uint64_t z_uid, z_gid; sa_bulk_attr_t bulk[8]; int count = 0; @@ -602,7 +593,6 @@ zfs_znode_alloc(zfs_sb_t *zsb, dmu_buf_t *db, int blksz, ASSERT(zp->z_dirlocks == NULL); ASSERT3P(zp->z_acl_cached, ==, NULL); ASSERT3P(zp->z_xattr_cached, ==, NULL); - ASSERT3P(zp->z_xattr_parent, ==, NULL); zp->z_moved = 0; zp->z_sa_hdl = NULL; zp->z_unlinked = 0; @@ -624,13 +614,13 @@ zfs_znode_alloc(zfs_sb_t *zsb, dmu_buf_t *db, int blksz, SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zsb), NULL, &mode, 8); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GEN(zsb), NULL, &tmp_gen, 8); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zsb), NULL, &zp->z_size, 8); - SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zsb), NULL, &zp->z_links, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zsb), NULL, &links, 8); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zsb), NULL, &zp->z_pflags, 8); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_PARENT(zsb), NULL, &parent, 8); - SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zsb), NULL, &zp->z_uid, 8); - SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zsb), NULL, &zp->z_gid, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zsb), NULL, &z_uid, 8); + SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zsb), NULL, &z_gid, 8); if (sa_bulk_lookup(zp->z_sa_hdl, bulk, count) != 0 || tmp_gen == 0) { @@ -643,14 +633,10 @@ zfs_znode_alloc(zfs_sb_t *zsb, dmu_buf_t *db, int blksz, zp->z_mode = mode; ip->i_generation = (uint32_t)tmp_gen; - - /* - * xattr znodes hold a reference on their unique parent - */ - if (dip && zp->z_pflags & ZFS_XATTR) { - igrab(dip); - zp->z_xattr_parent = ITOZ(dip); - } + ip->i_blkbits = SPA_MINBLOCKSHIFT; + set_nlink(ip, (uint32_t)links); + zfs_uid_write(ip, z_uid); + zfs_gid_write(ip, z_gid); ip->i_ino = obj; zfs_inode_update_new(zp); @@ -728,6 +714,7 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr, timestruc_t now; uint64_t gen, obj; int bonuslen; + int dnodesize; sa_handle_t *sa_hdl; dmu_object_type_t obj_type; sa_bulk_attr_t *sa_attrs; @@ -739,15 +726,21 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr, obj = vap->va_nodeid; now = vap->va_ctime; /* see zfs_replay_create() */ gen = vap->va_nblocks; /* ditto */ + dnodesize = vap->va_fsid; /* ditto */ } else { obj = 0; gethrestime(&now); gen = dmu_tx_get_txg(tx); + dnodesize = dmu_objset_dnodesize(zsb->z_os); } + if (dnodesize == 0) + dnodesize = DNODE_MIN_SIZE; + obj_type = zsb->z_use_sa ? DMU_OT_SA : DMU_OT_ZNODE; + bonuslen = (obj_type == DMU_OT_SA) ? - DN_MAX_BONUSLEN : ZFS_OLD_ZNODE_PHYS_SIZE; + DN_BONUS_SIZE(dnodesize) : ZFS_OLD_ZNODE_PHYS_SIZE; /* * Create a new DMU object. @@ -760,23 +753,23 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr, */ if (S_ISDIR(vap->va_mode)) { if (zsb->z_replay) { - VERIFY0(zap_create_claim_norm(zsb->z_os, obj, + VERIFY0(zap_create_claim_norm_dnsize(zsb->z_os, obj, zsb->z_norm, DMU_OT_DIRECTORY_CONTENTS, - obj_type, bonuslen, tx)); + obj_type, bonuslen, dnodesize, tx)); } else { - obj = zap_create_norm(zsb->z_os, + obj = zap_create_norm_dnsize(zsb->z_os, zsb->z_norm, DMU_OT_DIRECTORY_CONTENTS, - obj_type, bonuslen, tx); + obj_type, bonuslen, dnodesize, tx); } } else { if (zsb->z_replay) { - VERIFY0(dmu_object_claim(zsb->z_os, obj, + VERIFY0(dmu_object_claim_dnsize(zsb->z_os, obj, DMU_OT_PLAIN_FILE_CONTENTS, 0, - obj_type, bonuslen, tx)); + obj_type, bonuslen, dnodesize, tx)); } else { - obj = dmu_object_alloc(zsb->z_os, + obj = dmu_object_alloc_dnsize(zsb->z_os, DMU_OT_PLAIN_FILE_CONTENTS, 0, - obj_type, bonuslen, tx); + obj_type, bonuslen, dnodesize, tx); } } @@ -807,9 +800,10 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr, if (S_ISDIR(vap->va_mode)) { size = 2; /* contents ("." and "..") */ - links = (flag & (IS_ROOT_NODE | IS_XATTR)) ? 2 : 1; + links = 2; } else { - size = links = 0; + size = 0; + links = 1; } if (S_ISBLK(vap->va_mode) || S_ISCHR(vap->va_mode)) @@ -932,8 +926,7 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr, VERIFY(sa_replace_all_by_template(sa_hdl, sa_attrs, cnt, tx) == 0); if (!(flag & IS_ROOT_NODE)) { - *zpp = zfs_znode_alloc(zsb, db, 0, obj_type, obj, sa_hdl, - ZTOI(dzp)); + *zpp = zfs_znode_alloc(zsb, db, 0, obj_type, obj, sa_hdl); VERIFY(*zpp != NULL); VERIFY(dzp != NULL); } else { @@ -948,6 +941,7 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr, (*zpp)->z_pflags = pflags; (*zpp)->z_mode = mode; + (*zpp)->z_dnodesize = dnodesize; if (obj_type == DMU_OT_ZNODE || acl_ids->z_aclp->z_version < ZFS_ACL_VERSION_FUID) { @@ -1143,7 +1137,7 @@ zfs_zget(zfs_sb_t *zsb, uint64_t obj_num, znode_t **zpp) * bonus buffer. */ zp = zfs_znode_alloc(zsb, db, doi.doi_data_block_size, - doi.doi_bonus_type, obj_num, NULL, NULL); + doi.doi_bonus_type, obj_num, NULL); if (zp == NULL) { err = SET_ERROR(ENOENT); } else { @@ -1161,12 +1155,24 @@ zfs_rezget(znode_t *zp) dmu_buf_t *db; uint64_t obj_num = zp->z_id; uint64_t mode; + uint64_t links; sa_bulk_attr_t bulk[7]; int err; int count = 0; uint64_t gen; + uint64_t z_uid, z_gid; znode_hold_t *zh; + /* + * skip ctldir, otherwise they will always get invalidated. This will + * cause funny behaviour for the mounted snapdirs. Especially for + * Linux >= 3.18, d_invalidate will detach the mountpoint and prevent + * anyone automount it again as long as someone is still using the + * detached mount. + */ + if (zp->z_is_ctldir) + return (0); + zh = zfs_znode_hold_enter(zsb, obj_num); mutex_enter(&zp->z_acl_lock); @@ -1181,11 +1187,6 @@ zfs_rezget(znode_t *zp) nvlist_free(zp->z_xattr_cached); zp->z_xattr_cached = NULL; } - - if (zp->z_xattr_parent) { - zfs_iput_async(ZTOI(zp->z_xattr_parent)); - zp->z_xattr_parent = NULL; - } rw_exit(&zp->z_xattr_lock); ASSERT(zp->z_sa_hdl == NULL); @@ -1213,13 +1214,13 @@ zfs_rezget(znode_t *zp) SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zsb), NULL, &zp->z_size, sizeof (zp->z_size)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_LINKS(zsb), NULL, - &zp->z_links, sizeof (zp->z_links)); + &links, sizeof (links)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zsb), NULL, &zp->z_pflags, sizeof (zp->z_pflags)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_UID(zsb), NULL, - &zp->z_uid, sizeof (zp->z_uid)); + &z_uid, sizeof (z_uid)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_GID(zsb), NULL, - &zp->z_gid, sizeof (zp->z_gid)); + &z_gid, sizeof (z_gid)); SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MODE(zsb), NULL, &mode, sizeof (mode)); @@ -1230,6 +1231,8 @@ zfs_rezget(znode_t *zp) } zp->z_mode = mode; + zfs_uid_write(ZTOI(zp), z_uid); + zfs_gid_write(ZTOI(zp), z_gid); if (gen != ZTOI(zp)->i_generation) { zfs_znode_dmu_fini(zp); @@ -1237,7 +1240,9 @@ zfs_rezget(znode_t *zp) return (SET_ERROR(EIO)); } - zp->z_unlinked = (zp->z_links == 0); + zp->z_unlinked = (ZTOI(zp)->i_nlink == 0); + set_nlink(ZTOI(zp), (uint32_t)links); + zp->z_blksz = doi.doi_data_block_size; zp->z_atime_dirty = 0; zfs_inode_update_new(zp); @@ -1756,6 +1761,14 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) DMU_OT_NONE, 0, tx); ASSERT(error == 0); + /* + * Give dmu_object_alloc() a hint about where to start + * allocating new objects. Otherwise, since the metadnode's + * dnode_phys_t structure isn't initialized yet, dmu_object_next() + * would fail and we'd have to skip to the next dnode block. + */ + os->os_obj_next = moid + 1; + /* * Set starting attributes. */ diff --git a/module/zfs/zil.c b/module/zfs/zil.c index 289b23c7f488..863ccb930c37 100644 --- a/module/zfs/zil.c +++ b/module/zfs/zil.c @@ -1372,7 +1372,8 @@ zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) itxg->itxg_sod += itx->itx_sod; } else { avl_tree_t *t = &itxs->i_async_tree; - uint64_t foid = ((lr_ooo_t *)&itx->itx_lr)->lr_foid; + uint64_t foid = + LR_FOID_GET_OBJ(((lr_ooo_t *)&itx->itx_lr)->lr_foid); itx_async_node_t *ian; avl_index_t where; @@ -1918,7 +1919,8 @@ zil_close(zilog_t *zilog) mutex_exit(&zilog->zl_lock); if (txg) txg_wait_synced(zilog->zl_dmu_pool, txg); - ASSERT(!zilog_is_dirty(zilog)); + if (txg < spa_freeze_txg(zilog->zl_spa)) + ASSERT(!zilog_is_dirty(zilog)); taskq_destroy(zilog->zl_clean_taskq); zilog->zl_clean_taskq = NULL; @@ -2078,7 +2080,7 @@ typedef struct zil_replay_arg { static int zil_replay_error(zilog_t *zilog, lr_t *lr, int error) { - char name[MAXNAMELEN]; + char name[ZFS_MAX_DATASET_NAME_LEN]; zilog->zl_replaying_seq--; /* didn't actually replay this one */ @@ -2122,7 +2124,7 @@ zil_replay_log_record(zilog_t *zilog, lr_t *lr, void *zra, uint64_t claim_txg) */ if (TX_OOO(txtype)) { error = dmu_object_info(zilog->zl_os, - ((lr_ooo_t *)lr)->lr_foid, NULL); + LR_FOID_GET_OBJ(((lr_ooo_t *)lr)->lr_foid), NULL); if (error == ENOENT || error == EEXIST) return (0); } diff --git a/module/zfs/zio.c b/module/zfs/zio.c index 523a924d67b0..3dd8cffe9ca2 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2011, 2015 by Delphix. All rights reserved. + * Copyright (c) 2011, 2016 by Delphix. All rights reserved. * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved. */ @@ -40,6 +40,7 @@ #include #include #include +#include /* * ========================================================================== @@ -142,12 +143,21 @@ zio_init(void) */ if (arc_watch && !IS_P2ALIGNED(size, PAGESIZE)) continue; -#endif + /* + * Here's the problem - on 4K native devices in userland on + * Linux using O_DIRECT, buffers must be 4K aligned or I/O + * will fail with EINVAL, causing zdb (and others) to coredump. + * Since userland probably doesn't need optimized buffer caches, + * we just force 4K alignment on everything. + */ + align = 8 * SPA_MINBLOCKSIZE; +#else if (size <= 4 * SPA_MINBLOCKSIZE) { align = SPA_MINBLOCKSIZE; } else if (IS_P2ALIGNED(size, p2 >> 2)) { align = MIN(p2 >> 2, PAGESIZE); } +#endif if (align != 0) { char name[36]; @@ -736,9 +746,10 @@ zio_read(zio_t *pio, spa_t *spa, const blkptr_t *bp, zio_t * zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, void *data, uint64_t size, const zio_prop_t *zp, - zio_done_func_t *ready, zio_done_func_t *physdone, zio_done_func_t *done, - void *private, - zio_priority_t priority, enum zio_flag flags, const zbookmark_phys_t *zb) + zio_done_func_t *ready, zio_done_func_t *children_ready, + zio_done_func_t *physdone, zio_done_func_t *done, + void *private, zio_priority_t priority, enum zio_flag flags, + const zbookmark_phys_t *zb) { zio_t *zio; @@ -757,6 +768,7 @@ zio_write(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, ZIO_DDT_CHILD_WRITE_PIPELINE : ZIO_WRITE_PIPELINE); zio->io_ready = ready; + zio->io_children_ready = children_ready; zio->io_physdone = physdone; zio->io_prop = *zp; @@ -1131,6 +1143,16 @@ zio_write_bp_init(zio_t *zio) if (!IO_IS_ALLOCATING(zio)) return (ZIO_PIPELINE_CONTINUE); + if (zio->io_children_ready != NULL) { + /* + * Now that all our children are ready, run the callback + * associated with this zio in case it wants to modify the + * data to be written. + */ + ASSERT3U(zp->zp_level, >, 0); + zio->io_children_ready(zio); + } + ASSERT(zio->io_child_type != ZIO_CHILD_DDT); if (zio->io_bp_override) { @@ -1390,6 +1412,76 @@ zio_interrupt(zio_t *zio) zio_taskq_dispatch(zio, ZIO_TASKQ_INTERRUPT, B_FALSE); } +void +zio_delay_interrupt(zio_t *zio) +{ + /* + * The timeout_generic() function isn't defined in userspace, so + * rather than trying to implement the function, the zio delay + * functionality has been disabled for userspace builds. + */ + +#ifdef _KERNEL + /* + * If io_target_timestamp is zero, then no delay has been registered + * for this IO, thus jump to the end of this function and "skip" the + * delay; issuing it directly to the zio layer. + */ + if (zio->io_target_timestamp != 0) { + hrtime_t now = gethrtime(); + + if (now >= zio->io_target_timestamp) { + /* + * This IO has already taken longer than the target + * delay to complete, so we don't want to delay it + * any longer; we "miss" the delay and issue it + * directly to the zio layer. This is likely due to + * the target latency being set to a value less than + * the underlying hardware can satisfy (e.g. delay + * set to 1ms, but the disks take 10ms to complete an + * IO request). + */ + + DTRACE_PROBE2(zio__delay__miss, zio_t *, zio, + hrtime_t, now); + + zio_interrupt(zio); + } else { + taskqid_t tid; + hrtime_t diff = zio->io_target_timestamp - now; + clock_t expire_at_tick = ddi_get_lbolt() + + NSEC_TO_TICK(diff); + + DTRACE_PROBE3(zio__delay__hit, zio_t *, zio, + hrtime_t, now, hrtime_t, diff); + + if (NSEC_TO_TICK(diff) == 0) { + /* Our delay is less than a jiffy - just spin */ + zfs_sleep_until(zio->io_target_timestamp); + } else { + /* + * Use taskq_dispatch_delay() in the place of + * OpenZFS's timeout_generic(). + */ + tid = taskq_dispatch_delay(system_taskq, + (task_func_t *) zio_interrupt, + zio, TQ_NOSLEEP, expire_at_tick); + if (!tid) { + /* + * Couldn't allocate a task. Just + * finish the zio without a delay. + */ + zio_interrupt(zio); + } + } + } + return; + } +#endif + DTRACE_PROBE1(zio__delay__skip, zio_t *, zio); + zio_interrupt(zio); +} + /* * Execute the I/O pipeline until one of the following occurs: * (1) the I/O completes; (2) the pipeline stalls waiting for @@ -2072,9 +2164,9 @@ zio_write_gang_block(zio_t *pio) zio_nowait(zio_write(zio, spa, txg, &gbh->zg_blkptr[g], (char *)pio->io_data + (pio->io_size - resid), lsize, &zp, - zio_write_gang_member_ready, NULL, NULL, &gn->gn_child[g], - pio->io_priority, ZIO_GANG_CHILD_FLAGS(pio), - &pio->io_bookmark)); + zio_write_gang_member_ready, NULL, NULL, NULL, + &gn->gn_child[g], pio->io_priority, + ZIO_GANG_CHILD_FLAGS(pio), &pio->io_bookmark)); } /* @@ -2457,7 +2549,7 @@ zio_ddt_write(zio_t *zio) dio = zio_write(zio, spa, txg, bp, zio->io_orig_data, zio->io_orig_size, &czp, NULL, NULL, - zio_ddt_ditto_write_done, dde, zio->io_priority, + NULL, zio_ddt_ditto_write_done, dde, zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); zio_push_transform(dio, zio->io_data, zio->io_size, 0, NULL); @@ -2478,7 +2570,8 @@ zio_ddt_write(zio_t *zio) ddt_phys_addref(ddp); } else { cio = zio_write(zio, spa, txg, bp, zio->io_orig_data, - zio->io_orig_size, zp, zio_ddt_child_write_ready, NULL, + zio->io_orig_size, zp, + zio_ddt_child_write_ready, NULL, NULL, zio_ddt_child_write_done, dde, zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark); diff --git a/module/zfs/zio_inject.c b/module/zfs/zio_inject.c index 1458be477280..61b7d25e6536 100644 --- a/module/zfs/zio_inject.c +++ b/module/zfs/zio_inject.c @@ -20,7 +20,7 @@ */ /* * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2012, 2014 by Delphix. All rights reserved. + * Copyright (c) 2012, 2015 by Delphix. All rights reserved. */ /* @@ -49,15 +49,53 @@ uint32_t zio_injection_enabled = 0; +/* + * Data describing each zinject handler registered on the system, and + * contains the list node linking the handler in the global zinject + * handler list. + */ typedef struct inject_handler { int zi_id; spa_t *zi_spa; zinject_record_t zi_record; + uint64_t *zi_lanes; + int zi_next_lane; list_node_t zi_link; } inject_handler_t; +/* + * List of all zinject handlers registered on the system, protected by + * the inject_lock defined below. + */ static list_t inject_handlers; + +/* + * This protects insertion into, and traversal of, the inject handler + * list defined above; as well as the inject_delay_count. Any time a + * handler is inserted or removed from the list, this lock should be + * taken as a RW_WRITER; and any time traversal is done over the list + * (without modification to it) this lock should be taken as a RW_READER. + */ static krwlock_t inject_lock; + +/* + * This holds the number of zinject delay handlers that have been + * registered on the system. It is protected by the inject_lock defined + * above. Thus modifications to this count must be a RW_WRITER of the + * inject_lock, and reads of this count must be (at least) a RW_READER + * of the lock. + */ +static int inject_delay_count = 0; + +/* + * This lock is used only in zio_handle_io_delay(), refer to the comment + * in that function for more details. + */ +static kmutex_t inject_delay_mtx; + +/* + * Used to assign unique identifying numbers to each new zinject handler. + */ static int inject_next_id = 1; /* @@ -361,21 +399,70 @@ spa_handle_ignored_writes(spa_t *spa) rw_exit(&inject_lock); } -uint64_t +hrtime_t zio_handle_io_delay(zio_t *zio) { vdev_t *vd = zio->io_vd; + inject_handler_t *min_handler = NULL; + hrtime_t min_target = 0; inject_handler_t *handler; - uint64_t seconds = 0; - - if (zio_injection_enabled == 0) - return (0); + hrtime_t idle; + hrtime_t busy; + hrtime_t target; rw_enter(&inject_lock, RW_READER); - for (handler = list_head(&inject_handlers); handler != NULL; - handler = list_next(&inject_handlers, handler)) { + /* + * inject_delay_count is a subset of zio_injection_enabled that + * is only incremented for delay handlers. These checks are + * mainly added to remind the reader why we're not explicitly + * checking zio_injection_enabled like the other functions. + */ + IMPLY(inject_delay_count > 0, zio_injection_enabled > 0); + IMPLY(zio_injection_enabled == 0, inject_delay_count == 0); + + /* + * If there aren't any inject delay handlers registered, then we + * can short circuit and simply return 0 here. A value of zero + * informs zio_delay_interrupt() that this request should not be + * delayed. This short circuit keeps us from acquiring the + * inject_delay_mutex unnecessarily. + */ + if (inject_delay_count == 0) { + rw_exit(&inject_lock); + return (0); + } + + /* + * Each inject handler has a number of "lanes" associated with + * it. Each lane is able to handle requests independently of one + * another, and at a latency defined by the inject handler + * record's zi_timer field. Thus if a handler in configured with + * a single lane with a 10ms latency, it will delay requests + * such that only a single request is completed every 10ms. So, + * if more than one request is attempted per each 10ms interval, + * the average latency of the requests will be greater than + * 10ms; but if only a single request is submitted each 10ms + * interval the average latency will be 10ms. + * + * We need to acquire this mutex to prevent multiple concurrent + * threads being assigned to the same lane of a given inject + * handler. The mutex allows us to perform the following two + * operations atomically: + * + * 1. determine the minimum handler and minimum target + * value of all the possible handlers + * 2. update that minimum handler's lane array + * + * Without atomicity, two (or more) threads could pick the same + * lane in step (1), and then conflict with each other in step + * (2). This could allow a single lane handler to process + * multiple requests simultaneously, which shouldn't be possible. + */ + mutex_enter(&inject_delay_mtx); + for (handler = list_head(&inject_handlers); + handler != NULL; handler = list_next(&inject_handlers, handler)) { if (handler->zi_record.zi_cmd != ZINJECT_DELAY_IO) continue; @@ -384,14 +471,101 @@ zio_handle_io_delay(zio_t *zio) continue; } - if (vd->vdev_guid == handler->zi_record.zi_guid) { - seconds = handler->zi_record.zi_timer; - break; + if (vd->vdev_guid != handler->zi_record.zi_guid) + continue; + + /* + * Defensive; should never happen as the array allocation + * occurs prior to inserting this handler on the list. + */ + ASSERT3P(handler->zi_lanes, !=, NULL); + + /* + * This should never happen, the zinject command should + * prevent a user from setting an IO delay with zero lanes. + */ + ASSERT3U(handler->zi_record.zi_nlanes, !=, 0); + + ASSERT3U(handler->zi_record.zi_nlanes, >, + handler->zi_next_lane); + + /* + * We want to issue this IO to the lane that will become + * idle the soonest, so we compare the soonest this + * specific handler can complete the IO with all other + * handlers, to find the lowest value of all possible + * lanes. We then use this lane to submit the request. + * + * Since each handler has a constant value for its + * delay, we can just use the "next" lane for that + * handler; as it will always be the lane with the + * lowest value for that particular handler (i.e. the + * lane that will become idle the soonest). This saves a + * scan of each handler's lanes array. + * + * There's two cases to consider when determining when + * this specific IO request should complete. If this + * lane is idle, we want to "submit" the request now so + * it will complete after zi_timer milliseconds. Thus, + * we set the target to now + zi_timer. + * + * If the lane is busy, we want this request to complete + * zi_timer milliseconds after the lane becomes idle. + * Since the 'zi_lanes' array holds the time at which + * each lane will become idle, we use that value to + * determine when this request should complete. + */ + idle = handler->zi_record.zi_timer + gethrtime(); + busy = handler->zi_record.zi_timer + + handler->zi_lanes[handler->zi_next_lane]; + target = MAX(idle, busy); + + if (min_handler == NULL) { + min_handler = handler; + min_target = target; + continue; } + ASSERT3P(min_handler, !=, NULL); + ASSERT3U(min_target, !=, 0); + + /* + * We don't yet increment the "next lane" variable since + * we still might find a lower value lane in another + * handler during any remaining iterations. Once we're + * sure we've selected the absolute minimum, we'll claim + * the lane and increment the handler's "next lane" + * field below. + */ + + if (target < min_target) { + min_handler = handler; + min_target = target; + } } + + /* + * 'min_handler' will be NULL if no IO delays are registered for + * this vdev, otherwise it will point to the handler containing + * the lane that will become idle the soonest. + */ + if (min_handler != NULL) { + ASSERT3U(min_target, !=, 0); + min_handler->zi_lanes[min_handler->zi_next_lane] = min_target; + + /* + * If we've used all possible lanes for this handler, + * loop back and start using the first lane again; + * otherwise, just increment the lane index. + */ + min_handler->zi_next_lane = (min_handler->zi_next_lane + 1) % + min_handler->zi_record.zi_nlanes; + } + + mutex_exit(&inject_delay_mtx); rw_exit(&inject_lock); - return (seconds); + + return (min_target); } /* @@ -415,6 +589,24 @@ zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record) if ((error = spa_reset(name)) != 0) return (error); + if (record->zi_cmd == ZINJECT_DELAY_IO) { + /* + * A value of zero for the number of lanes or for the + * delay time doesn't make sense. + */ + if (record->zi_timer == 0 || record->zi_nlanes == 0) + return (SET_ERROR(EINVAL)); + + /* + * The number of lanes is directly mapped to the size of + * an array used by the handler. Thus, to ensure the + * user doesn't trigger an allocation that's "too large" + * we cap the number of lanes here. + */ + if (record->zi_nlanes >= UINT16_MAX) + return (SET_ERROR(EINVAL)); + } + if (!(flags & ZINJECT_NULL)) { /* * spa_inject_ref() will add an injection reference, which will @@ -426,11 +618,34 @@ zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record) handler = kmem_alloc(sizeof (inject_handler_t), KM_SLEEP); + handler->zi_spa = spa; + handler->zi_record = *record; + + if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) { + handler->zi_lanes = kmem_zalloc( + sizeof (*handler->zi_lanes) * + handler->zi_record.zi_nlanes, KM_SLEEP); + handler->zi_next_lane = 0; + } else { + handler->zi_lanes = NULL; + handler->zi_next_lane = 0; + } + rw_enter(&inject_lock, RW_WRITER); + /* + * We can't move this increment into the conditional + * above because we need to hold the RW_WRITER lock of + * inject_lock, and we don't want to hold that while + * allocating the handler's zi_lanes array. + */ + if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) { + ASSERT3S(inject_delay_count, >=, 0); + inject_delay_count++; + ASSERT3S(inject_delay_count, >, 0); + } + *id = handler->zi_id = inject_next_id++; - handler->zi_spa = spa; - handler->zi_record = *record; list_insert_tail(&inject_handlers, handler); atomic_inc_32(&zio_injection_enabled); @@ -508,9 +723,23 @@ zio_clear_fault(int id) return (SET_ERROR(ENOENT)); } + if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) { + ASSERT3S(inject_delay_count, >, 0); + inject_delay_count--; + ASSERT3S(inject_delay_count, >=, 0); + } + list_remove(&inject_handlers, handler); rw_exit(&inject_lock); + if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) { + ASSERT3P(handler->zi_lanes, !=, NULL); + kmem_free(handler->zi_lanes, sizeof (*handler->zi_lanes) * + handler->zi_record.zi_nlanes); + } else { + ASSERT3P(handler->zi_lanes, ==, NULL); + } + spa_inject_delref(handler->zi_spa); kmem_free(handler, sizeof (inject_handler_t)); atomic_dec_32(&zio_injection_enabled); @@ -522,6 +751,7 @@ void zio_inject_init(void) { rw_init(&inject_lock, NULL, RW_DEFAULT, NULL); + mutex_init(&inject_delay_mtx, NULL, MUTEX_DEFAULT, NULL); list_create(&inject_handlers, sizeof (inject_handler_t), offsetof(inject_handler_t, zi_link)); } @@ -530,6 +760,7 @@ void zio_inject_fini(void) { list_destroy(&inject_handlers); + mutex_destroy(&inject_delay_mtx); rw_destroy(&inject_lock); } diff --git a/module/zfs/zpl_inode.c b/module/zfs/zpl_inode.c index 089e3a1bcc97..8c75698e5881 100644 --- a/module/zfs/zpl_inode.c +++ b/module/zfs/zpl_inode.c @@ -50,7 +50,7 @@ zpl_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags) int zfs_flags = 0; zfs_sb_t *zsb = dentry->d_sb->s_fs_info; - if (dlen(dentry) > ZFS_MAXNAMELEN) + if (dlen(dentry) > ZFS_MAX_DATASET_NAME_LEN) return (ERR_PTR(-ENAMETOOLONG)); crhold(cr); diff --git a/module/zfs/zpl_super.c b/module/zfs/zpl_super.c index bcdbbd69e280..91c36c9e3675 100644 --- a/module/zfs/zpl_super.c +++ b/module/zfs/zpl_super.c @@ -336,12 +336,12 @@ zpl_parse_options(char *osname, char *mntopts, zfs_mntopts_t *zmo, if (mntopts) { substring_t args[MAX_OPT_ARGS]; - char *tmp_mntopts, *p; + char *tmp_mntopts, *p, *t; int token; - tmp_mntopts = strdup(mntopts); + t = tmp_mntopts = strdup(mntopts); - while ((p = strsep(&tmp_mntopts, ",")) != NULL) { + while ((p = strsep(&t, ",")) != NULL) { if (!*p) continue; diff --git a/module/zfs/zvol.c b/module/zfs/zvol.c index 9c89493edfcf..c25b243db3a8 100644 --- a/module/zfs/zvol.c +++ b/module/zfs/zvol.c @@ -61,7 +61,7 @@ unsigned long zvol_max_discard_blocks = 16384; static kmutex_t zvol_state_lock; static list_t zvol_state_list; -static char *zvol_tag = "zvol_tag"; +void *zvol_tag = "zvol_tag"; /* * The in-core state of each volume. @@ -812,7 +812,7 @@ zvol_request(struct request_queue *q, struct bio *bio) goto out2; } - if (bio->bi_rw & VDEV_REQ_DISCARD) { + if (bio_is_discard(bio)) { error = zvol_discard(bio); goto out2; } diff --git a/rpm/generic/zfs.spec.in b/rpm/generic/zfs.spec.in index cc8d02b6de78..7e851cf30a9a 100644 --- a/rpm/generic/zfs.spec.in +++ b/rpm/generic/zfs.spec.in @@ -84,6 +84,8 @@ Conflicts: zfs-fuse BuildRequires: zlib-devel BuildRequires: libuuid-devel BuildRequires: libblkid-devel +BuildRequires: libudev-devel +BuildRequires: libattr-devel %endif %if 0%{?_systemd} Requires(post): systemd diff --git a/scripts/common.sh.in b/scripts/common.sh.in index 27ba88fae766..010b534be668 100644 --- a/scripts/common.sh.in +++ b/scripts/common.sh.in @@ -12,7 +12,7 @@ if [ -f "${basedir}/../${SCRIPT_CONFIG}" ]; then . "${basedir}/../${SCRIPT_CONFIG}" else KERNEL_MODULES=(zlib_deflate zlib_inflate) -MODULES=(spl splat zavl znvpair zunicode zcommon zfs) +MODULES=(spl splat zavl znvpair zunicode zcommon icp zfs) fi PROG="" @@ -148,17 +148,6 @@ populate() { } init() { - # Disable the udev rule 90-zfs.rules to prevent the zfs module - # stack from being loaded due to the detection of a zfs device. - # This is important because the test scripts require full control - # over when and how the modules are loaded/unloaded. A trap is - # set to ensure the udev rule is correctly replaced on exit. - local RULE=${udevruledir}/90-zfs.rules - if test -e ${RULE}; then - trap "mv ${RULE}.disabled ${RULE}" INT TERM EXIT - mv ${RULE} ${RULE}.disabled - fi - # Create a random directory tree of files and sub-directories to # to act as a copy source for the various regression tests. SRC_DIR=`mktemp -d -p /var/tmp/ zfs.src.XXXXXXXX` diff --git a/scripts/cstyle.pl b/scripts/cstyle.pl index 4df185eb38ce..7baf2554530e 100755 --- a/scripts/cstyle.pl +++ b/scripts/cstyle.pl @@ -191,7 +191,11 @@ sub err($) { my ($error) = @_; unless ($no_errs) { - printf $fmt, $filename, $., $error, $line; + if ($verbose) { + printf $fmt, $filename, $., $error, $line; + } else { + printf $fmt, $filename, $., $error; + } $err_stat = 1; } } @@ -200,7 +204,11 @@ ($$) my ($prevline, $error) = @_; my $out = $prevline."\n".$line; unless ($no_errs) { - printf $fmt, $filename, $., $error, $out; + if ($verbose) { + printf $fmt, $filename, $., $error, $out; + } else { + printf $fmt, $filename, $., $error; + } $err_stat = 1; } } @@ -208,7 +216,11 @@ ($$) sub err_prev($) { my ($error) = @_; unless ($no_errs) { - printf $fmt, $filename, $. - 1, $error, $prev; + if ($verbose) { + printf $fmt, $filename, $. - 1, $error, $prev; + } else { + printf $fmt, $filename, $. - 1, $error; + } $err_stat = 1; } } @@ -605,7 +617,7 @@ ($$) if (/^\s*\(void\)[^ ]/) { err("missing space after (void) cast"); } - if (/\S{/ && !/{{/) { + if (/\S\{/ && !/\{\{/) { err("missing space before left brace"); } if ($in_function && /^\s+{/ && diff --git a/scripts/zconfig.sh b/scripts/zconfig.sh index 1908dc1d6919..c2b97c2c55e0 100755 --- a/scripts/zconfig.sh +++ b/scripts/zconfig.sh @@ -63,6 +63,17 @@ fi # Initialize the test suite init +# Disable the udev rule 90-zfs.rules to prevent the zfs module +# stack from being loaded due to the detection of a zfs device. +# This is important because this test scripts require full control +# over when and how the modules are loaded/unloaded. A trap is +# set to ensure the udev rule is correctly replaced on exit. +RULE=${udevruledir}/90-zfs.rules +if test -e ${RULE}; then + trap "mv ${RULE}.disabled ${RULE}" INT TERM EXIT + mv ${RULE} ${RULE}.disabled +fi + # Perform pre-cleanup is requested if [ ${CLEANUP} ]; then ${ZFS_SH} -u diff --git a/scripts/zloop.sh b/scripts/zloop.sh index 2550eab45ca5..8125229a8cb1 100755 --- a/scripts/zloop.sh +++ b/scripts/zloop.sh @@ -63,9 +63,25 @@ function or_die fi } +# core file helpers +origcorepattern="$(cat /proc/sys/kernel/core_pattern)" +coreglob="$(egrep -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" + +if [[ $coreglob = "*" ]]; then + echo "Setting core file pattern..." + echo "core" > /proc/sys/kernel/core_pattern + coreglob="$(egrep -o '^([^|%[:space:]]*)' /proc/sys/kernel/core_pattern)*" +fi + +function core_file +{ + printf "%s" "$(ls -tr1 $coreglob 2> /dev/null | head -1)" +} + function store_core { - if [[ $ztrc -ne 0 ]] || [[ -f core ]]; then + core="$(core_file)" + if [[ $ztrc -ne 0 ]] || [[ -f "$core" ]]; then coreid=$(date "+zloop-%y%m%d-%H%M%S") foundcrashes=$(($foundcrashes + 1)) @@ -82,7 +98,7 @@ function store_core or_die mv $workdir/zpool.cache $dest/vdev/ # check for core - if [[ -f core ]]; then + if [[ -f "$core" ]]; then corestatus=$($GDB --batch --quiet \ -ex "set print thread-events off" \ -ex "printf \"*\n* Backtrace \n*\n\"" \ @@ -95,14 +111,14 @@ function store_core -ex "thread apply all bt" \ -ex "printf \"*\n* Backtraces (full) \n*\n\"" \ -ex "thread apply all bt full" \ - -ex "quit" $ZTEST core | grep -v "New LWP") + -ex "quit" $ZTEST "$core" | grep -v "New LWP") # Dump core + logs to stored directory echo "$corestatus" >>$dest/status - or_die mv core $dest/ + or_die mv "$core" $dest/ # Record info in cores logfile - echo "*** core @ $coredir/$coreid/core:" | \ + echo "*** core @ $coredir/$coreid/$core:" | \ tee -a ztest.cores echo "$corestatus" | tee -a ztest.cores echo "" | tee -a ztest.cores @@ -135,8 +151,9 @@ shift $((OPTIND - 1)) # enable core dumps ulimit -c unlimited -if [[ -f core ]]; then - echo "There's a core dump here you might want to look at first." +if [[ -f "$(core_file)" ]]; then + echo -n "There's a core dump here you might want to look at first... " + echo "$(core_file)" exit 1 fi @@ -206,6 +223,9 @@ done echo "zloop finished, $foundcrashes crashes found" +#restore core pattern +echo "$origcorepattern" > /proc/sys/kernel/core_pattern + uptime >>ztest.out if [[ $foundcrashes -gt 0 ]]; then diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index c9b882987bad..42a82e4b3807 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -19,10 +19,11 @@ post_user = root post = cleanup outputdir = /var/tmp/test_results -# DISABLED: -# posix_001_pos - needs investigation +# DISABLED: update to use ZFS_ACL_* variables and user_run helper. +# posix_001_pos +# posix_002_pos [tests/functional/acl/posix] -tests = ['posix_002_pos', 'posix_003_pos'] +tests = ['posix_003_pos'] [tests/functional/atime] tests = ['atime_001_pos', 'atime_002_neg', 'atime_003_pos'] @@ -145,14 +146,13 @@ tests = ['zfs_promote_001_pos', 'zfs_promote_002_pos', 'zfs_promote_003_pos', tests = [] # DISABLED: -# zfs_receive_003_pos - needs investigation -# zfs_receive_010_pos - needs investigation -# zfs_receive_011_pos - needs investigation -# zfs_receive_012_pos - needs investigation +# zfs_receive_004_neg - Fails for OpenZFS on illumos +# zfs_receive_011_pos - Requires port of OpenZFS 6562 [tests/functional/cli_root/zfs_receive] -tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_005_neg', - 'zfs_receive_006_pos', 'zfs_receive_007_neg', 'zfs_receive_008_pos', - 'zfs_receive_009_neg'] +tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos', + 'zfs_receive_005_neg', 'zfs_receive_006_pos', + 'zfs_receive_007_neg', 'zfs_receive_008_pos', 'zfs_receive_009_neg', + 'zfs_receive_010_pos', 'zfs_receive_012_pos'] # DISABLED: # zfs_rename_002_pos - needs investigation @@ -174,11 +174,10 @@ tests = ['zfs_reservation_001_pos', 'zfs_reservation_002_pos'] [tests/functional/cli_root/zfs_rollback] tests = ['zfs_rollback_003_neg', 'zfs_rollback_004_neg'] -# DISABLED: -# zfs_send_007_pos - needs investigation [tests/functional/cli_root/zfs_send] tests = ['zfs_send_001_pos', 'zfs_send_002_pos', 'zfs_send_003_pos', - 'zfs_send_004_neg', 'zfs_send_005_pos', 'zfs_send_006_pos'] + 'zfs_send_004_neg', 'zfs_send_005_pos', 'zfs_send_006_pos', + 'zfs_send_007_pos'] # DISABLED: # mountpoint_003_pos - needs investigation @@ -206,10 +205,11 @@ tests = ['cache_001_pos', 'cache_002_neg', 'canmount_001_pos', # DISABLED: # zfs_snapshot_008_neg - nested pools +# zfs_snapshot_009_pos - Fails for OpenZFS on illumos [tests/functional/cli_root/zfs_snapshot] tests = ['zfs_snapshot_001_neg', 'zfs_snapshot_002_neg', 'zfs_snapshot_003_neg', 'zfs_snapshot_004_neg', 'zfs_snapshot_005_neg', - 'zfs_snapshot_006_pos', 'zfs_snapshot_007_neg', 'zfs_snapshot_009_pos'] + 'zfs_snapshot_006_pos', 'zfs_snapshot_007_neg'] # DISABLED: # zfs_unmount_005_pos - needs investigation @@ -261,7 +261,7 @@ tests = ['zpool_clear_001_pos', 'zpool_clear_002_neg', 'zpool_clear_003_neg'] # zpool_create_020_pos - needs investigation [tests/functional/cli_root/zpool_create] tests = [ - 'zpool_create_003_pos', 'zpool_create_005_pos', 'zpool_create_007_neg', + 'zpool_create_003_pos', 'zpool_create_005_pos', 'zpool_create_007_neg', 'zpool_create_009_neg', 'zpool_create_010_neg', 'zpool_create_017_neg', 'zpool_create_018_pos', 'zpool_create_019_pos', 'zpool_create_021_pos', 'zpool_create_022_pos', 'zpool_create_023_neg', @@ -339,41 +339,47 @@ post = [tests/functional/cli_root/zpool_status] tests = ['zpool_status_001_pos', 'zpool_status_002_pos'] -# DISABLED: ENOSPC failure -#[tests/functional/cli_root/zpool_upgrade] -#tests = ['zpool_upgrade_001_pos', 'zpool_upgrade_002_pos', -# 'zpool_upgrade_003_pos', 'zpool_upgrade_004_pos', 'zpool_upgrade_005_neg', -# 'zpool_upgrade_006_neg', 'zpool_upgrade_007_pos', 'zpool_upgrade_008_pos', -# 'zpool_upgrade_009_neg'] - -# DISABLED: nested pools -#[tests/functional/cli_user/misc] -#tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg', -# 'zfs_clone_001_neg', 'zfs_create_001_neg', 'zfs_destroy_001_neg', -# 'zfs_get_001_neg', 'zfs_inherit_001_neg', 'zfs_mount_001_neg', -# 'zfs_promote_001_neg', 'zfs_receive_001_neg', 'zfs_rename_001_neg', -# 'zfs_rollback_001_neg', 'zfs_send_001_neg', 'zfs_set_001_neg', -# 'zfs_share_001_neg', 'zfs_snapshot_001_neg', 'zfs_unallow_001_neg', -# 'zfs_unmount_001_neg', 'zfs_unshare_001_neg', 'zfs_upgrade_001_neg', -# 'zpool_001_neg', 'zpool_add_001_neg', 'zpool_attach_001_neg', -# 'zpool_clear_001_neg', 'zpool_create_001_neg', 'zpool_destroy_001_neg', -# 'zpool_detach_001_neg', 'zpool_export_001_neg', 'zpool_get_001_neg', -# 'zpool_history_001_neg', 'zpool_import_001_neg', 'zpool_import_002_neg', -# 'zpool_offline_001_neg', 'zpool_online_001_neg', 'zpool_remove_001_neg', -# 'zpool_replace_001_neg', 'zpool_scrub_001_neg', 'zpool_set_001_neg', -# 'zpool_status_001_neg', 'zpool_upgrade_001_neg'] -#user = zfs-tests +# DISABLED: +# zpool_upgrade_007_pos - needs investigation +[tests/functional/cli_root/zpool_upgrade] +tests = ['zpool_upgrade_001_pos', 'zpool_upgrade_002_pos', + 'zpool_upgrade_003_pos', 'zpool_upgrade_004_pos', 'zpool_upgrade_005_neg', + 'zpool_upgrade_006_neg', 'zpool_upgrade_008_pos', + 'zpool_upgrade_009_neg'] + +# DISABLED: +# zfs_share_001_neg - requires additional dependencies +# zfs_unshare_001_neg - requires additional dependencies +[tests/functional/cli_user/misc] +tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg', + 'zfs_clone_001_neg', 'zfs_create_001_neg', 'zfs_destroy_001_neg', + 'zfs_get_001_neg', 'zfs_inherit_001_neg', 'zfs_mount_001_neg', + 'zfs_promote_001_neg', 'zfs_receive_001_neg', 'zfs_rename_001_neg', + 'zfs_rollback_001_neg', 'zfs_send_001_neg', 'zfs_set_001_neg', + 'zfs_snapshot_001_neg', 'zfs_unallow_001_neg', + 'zfs_unmount_001_neg', 'zfs_upgrade_001_neg', + 'zpool_001_neg', 'zpool_add_001_neg', 'zpool_attach_001_neg', + 'zpool_clear_001_neg', 'zpool_create_001_neg', 'zpool_destroy_001_neg', + 'zpool_detach_001_neg', 'zpool_export_001_neg', 'zpool_get_001_neg', + 'zpool_history_001_neg', 'zpool_import_001_neg', 'zpool_import_002_neg', + 'zpool_offline_001_neg', 'zpool_online_001_neg', 'zpool_remove_001_neg', + 'zpool_replace_001_neg', 'zpool_scrub_001_neg', 'zpool_set_001_neg', + 'zpool_status_001_neg', 'zpool_upgrade_001_neg'] +user = [tests/functional/cli_user/zfs_list] tests = ['zfs_list_001_pos', 'zfs_list_002_pos', 'zfs_list_003_pos', 'zfs_list_004_neg', 'zfs_list_007_pos', 'zfs_list_008_neg'] +user = [tests/functional/cli_user/zpool_iostat] tests = ['zpool_iostat_001_neg', 'zpool_iostat_002_pos', 'zpool_iostat_003_neg', 'zpool_iostat_004_pos'] +user = [tests/functional/cli_user/zpool_list] tests = ['zpool_list_001_pos', 'zpool_list_002_neg'] +user = [tests/functional/compression] tests = ['compress_001_pos', 'compress_002_pos', 'compress_003_pos', @@ -382,15 +388,14 @@ tests = ['compress_001_pos', 'compress_002_pos', 'compress_003_pos', [tests/functional/ctime] tests = ['ctime_001_pos' ] -# DISABLED: Linux does not yet support delegations. -#[tests/functional/delegate] -#tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos', -# 'zfs_allow_004_pos', 'zfs_allow_005_pos', 'zfs_allow_006_pos', -# 'zfs_allow_007_pos', 'zfs_allow_008_pos', 'zfs_allow_009_neg', -# 'zfs_allow_010_pos', 'zfs_allow_011_neg', 'zfs_allow_012_neg', -# 'zfs_unallow_001_pos', 'zfs_unallow_002_pos', 'zfs_unallow_003_pos', -# 'zfs_unallow_004_pos', 'zfs_unallow_005_pos', 'zfs_unallow_006_pos', -# 'zfs_unallow_007_neg', 'zfs_unallow_008_neg'] +[tests/functional/delegate] +tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos', + 'zfs_allow_004_pos', 'zfs_allow_005_pos', 'zfs_allow_006_pos', + 'zfs_allow_007_pos', 'zfs_allow_008_pos', 'zfs_allow_009_neg', + 'zfs_allow_010_pos', 'zfs_allow_011_neg', 'zfs_allow_012_neg', + 'zfs_unallow_001_pos', 'zfs_unallow_002_pos', 'zfs_unallow_003_pos', + 'zfs_unallow_004_pos', 'zfs_unallow_005_pos', 'zfs_unallow_006_pos', + 'zfs_unallow_007_neg', 'zfs_unallow_008_neg'] # DISABLED: # devices_001_pos - needs investigation @@ -406,6 +411,11 @@ tests = ['exec_001_pos'] [tests/functional/features/async_destroy] tests = ['async_destroy_001_pos'] +[tests/functional/features/large_dnode] +tests = ['large_dnode_001_pos', 'large_dnode_002_pos', 'large_dnode_003_pos', + 'large_dnode_004_neg', 'large_dnode_005_pos', 'large_dnode_006_pos', + 'large_dnode_007_neg'] + # DISABLED: needs investigation #[tests/functional/grow_pool] #tests = ['grow_pool_001_pos'] @@ -418,16 +428,11 @@ tests = ['async_destroy_001_pos'] #pre = #post = -# DISABLED: -# history_001_pos - export commands missing from history -# history_003_pos - nested pool -# history_006_neg - needs investigation -# history_007_pos - needs investigation -# history_008_pos - needs investigation -# history_010_pos - needs investigation [tests/functional/history] -tests = ['history_002_pos', 'history_004_pos', 'history_005_neg', - 'history_009_pos'] +tests = ['history_001_pos', 'history_002_pos', 'history_003_pos', + 'history_004_pos', 'history_005_neg', 'history_006_neg', + 'history_007_pos', 'history_008_pos', 'history_009_pos', + 'history_010_pos'] [tests/functional/inheritance] tests = ['inherit_001_pos'] @@ -507,12 +512,15 @@ tests = ['poolversion_001_pos', 'poolversion_002_pos'] #tests = ['privilege_001_pos', 'privilege_002_pos'] # DISABLED: -# quota_002_pos - size is less than current used or reserved space +# quota_002_pos - size is less than current used or reserved space # quota_004_pos - size is less than current used or reserved space # quota_005_pos - size is less than current used or reserved space [tests/functional/quota] tests = ['quota_001_pos', 'quota_003_pos', 'quota_006_neg'] +[tests/functional/raidz] +tests = ['raidz_001_neg', 'raidz_002_pos'] + [tests/functional/redundancy] tests = ['redundancy_001_pos', 'redundancy_002_pos', 'redundancy_003_pos'] @@ -552,12 +560,17 @@ tests = ['reservation_001_pos', 'reservation_002_pos', 'reservation_003_pos', #[tests/functional/rootpool] #tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_neg'] -# DISABLED: Hangs on I/O for unclear reason. -#[tests/functional/rsend] -#tests = ['rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', -# 'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', -# 'rsend_009_pos', 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', -# 'rsend_013_pos'] +# DISABLED: +# rsend_008_pos - Fails for OpenZFS on illumos +# rsend_009_pos - Fails for OpenZFS on illumos +# rsend_020_pos - ASSERTs in dump_record() +[tests/functional/rsend] +tests = ['rsend_001_pos', 'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', + 'rsend_005_pos', 'rsend_006_pos', 'rsend_007_pos', + 'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', + 'rsend_013_pos', 'rsend_014_pos', + 'rsend_019_pos', + 'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos'] [tests/functional/scrub_mirror] tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos', @@ -573,17 +586,17 @@ tests = ['slog_001_pos', 'slog_002_pos', 'slog_003_pos', 'slog_004_pos', 'slog_009_neg', 'slog_010_neg', 'slog_011_neg'] # DISABLED: +# clone_001_pos - nested pools # rollback_003_pos - Hangs in unmount and spins. -# snapshot_013_pos - Hangs on I/O for unclear reason. -# snapshot_016_pos - .zfs mv/rmdir/mkdir disabled by default. -#[tests/functional/snapshot] -#tests = ['clone_001_pos', 'rollback_001_pos', 'rollback_002_pos', -# 'snapshot_001_pos', 'snapshot_002_pos', -# 'snapshot_003_pos', 'snapshot_004_pos', 'snapshot_005_pos', -# 'snapshot_006_pos', 'snapshot_007_pos', 'snapshot_008_pos', -# 'snapshot_009_pos', 'snapshot_010_pos', 'snapshot_011_pos', -# 'snapshot_012_pos', 'snapshot_014_pos', -# 'snapshot_015_pos', 'snapshot_017_pos'] +# snapshot_016_pos - Problem with automount +[tests/functional/snapshot] +tests = ['rollback_001_pos', 'rollback_002_pos', + 'snapshot_001_pos', 'snapshot_002_pos', + 'snapshot_003_pos', 'snapshot_004_pos', 'snapshot_005_pos', + 'snapshot_006_pos', 'snapshot_007_pos', 'snapshot_008_pos', + 'snapshot_009_pos', 'snapshot_010_pos', 'snapshot_011_pos', + 'snapshot_012_pos', 'snapshot_013_pos', 'snapshot_014_pos', + 'snapshot_015_pos', 'snapshot_017_pos'] [tests/functional/snapused] tests = ['snapused_001_pos', 'snapused_002_pos', 'snapused_003_pos', 'snapused_004_pos', 'snapused_005_pos'] diff --git a/tests/test-runner/cmd/test-runner.py b/tests/test-runner/cmd/test-runner.py index dd6a3c7b6724..ad1afff80647 100755 --- a/tests/test-runner/cmd/test-runner.py +++ b/tests/test-runner/cmd/test-runner.py @@ -158,6 +158,10 @@ def update_cmd_privs(self, cmd, user): me = getpwuid(os.getuid()) if not user or user is me: + if os.path.isfile(cmd+'.ksh') and os.access(cmd+'.ksh', os.X_OK): + cmd += '.ksh' + if os.path.isfile(cmd+'.sh') and os.access(cmd+'.sh', os.X_OK): + cmd += '.sh' return cmd if not os.path.isfile(cmd): @@ -207,10 +211,11 @@ def run(self, options): except OSError, e: fail('%s' % e) + self.result.starttime = time() + proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) + t = Timer(int(self.timeout), self.kill_cmd, [proc]) + try: - self.result.starttime = time() - proc = Popen(privcmd, stdout=PIPE, stderr=PIPE) - t = Timer(int(self.timeout), self.kill_cmd, [proc]) t.start() self.result.stdout, self.result.stderr = self.collect_output(proc) except KeyboardInterrupt: diff --git a/tests/zfs-tests/cmd/file_write/file_write.c b/tests/zfs-tests/cmd/file_write/file_write.c index 43fd96ac465a..046794820589 100644 --- a/tests/zfs-tests/cmd/file_write/file_write.c +++ b/tests/zfs-tests/cmd/file_write/file_write.c @@ -127,7 +127,10 @@ main(int argc, char **argv) err++; } - if (err) usage(prog); + if (err) { + usage(prog); /* no return */ + return (1); + } /* * Prepare the buffer and determine the requested operation diff --git a/tests/zfs-tests/cmd/mktree/mktree.c b/tests/zfs-tests/cmd/mktree/mktree.c index 8f9b38578c11..95d31a6527d1 100644 --- a/tests/zfs-tests/cmd/mktree/mktree.c +++ b/tests/zfs-tests/cmd/mktree/mktree.c @@ -172,7 +172,7 @@ crtfile(char *pname) exit(errno); } - if (fsetxattr(fd, "xattr", pbuf, 1024, 0) < 0) { + if (fsetxattr(fd, "user.xattr", pbuf, 1024, 0) < 0) { (void) fprintf(stderr, "fsetxattr(fd, \"xattr\", pbuf, " "1024, 0) failed.\n[%d]: %s.\n", errno, strerror(errno)); exit(errno); diff --git a/tests/zfs-tests/cmd/randfree_file/randfree_file.c b/tests/zfs-tests/cmd/randfree_file/randfree_file.c index 8e7487c41e77..ff30c24c09dc 100644 --- a/tests/zfs-tests/cmd/randfree_file/randfree_file.c +++ b/tests/zfs-tests/cmd/randfree_file/randfree_file.c @@ -83,23 +83,26 @@ main(int argc, char *argv[]) else usage(argv[0]); - buf = (char *)malloc(filesize); - if ((fd = open(filename, O_RDWR | O_CREAT | O_TRUNC, mode)) < 0) { perror("open"); return (1); } + + buf = (char *)malloc(filesize); + if (write(fd, buf, filesize) < filesize) { + free(buf); perror("write"); return (1); } + free(buf); + if (fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, start_off, off_len) < 0) { perror("fallocate"); return (1); } - free(buf); return (0); } diff --git a/tests/zfs-tests/cmd/xattrtest/xattrtest.c b/tests/zfs-tests/cmd/xattrtest/xattrtest.c index 22f798db26f3..6b07d141ae41 100644 --- a/tests/zfs-tests/cmd/xattrtest/xattrtest.c +++ b/tests/zfs-tests/cmd/xattrtest/xattrtest.c @@ -51,7 +51,7 @@ extern char *program_invocation_short_name; program_invocation_short_name, __FILE__, __LINE__, \ __func__, ## __VA_ARGS__); -static const char shortopts[] = "hvycdn:f:x:s:p:t:e:rRk"; +static const char shortopts[] = "hvycdn:f:x:s:p:t:e:rRko:"; static const struct option longopts[] = { { "help", no_argument, 0, 'h' }, { "verbose", no_argument, 0, 'v' }, @@ -68,9 +68,19 @@ static const struct option longopts[] = { { "random", no_argument, 0, 'r' }, { "randomvalue", no_argument, 0, 'R' }, { "keep", no_argument, 0, 'k' }, + { "only", required_argument, 0, 'o' }, { 0, 0, 0, 0 } }; +enum phases { + PHASE_ALL = 0, + PHASE_CREATE, + PHASE_SETXATTR, + PHASE_GETXATTR, + PHASE_UNLINK, + PHASE_INVAL +}; + static int verbose = 0; static int verify = 0; static int synccaches = 0; @@ -78,18 +88,22 @@ static int dropcaches = 0; static int nth = 0; static int files = 1000; static int xattrs = 1; -static int size = 1; +static int size = 6; static int size_is_random = 0; static int value_is_random = 0; static int keep_files = 0; +static int phase = PHASE_ALL; static char path[PATH_MAX] = "/tmp/xattrtest"; static char script[PATH_MAX] = "/bin/true"; +static char xattrbytes[XATTR_SIZE_MAX]; static int usage(int argc, char **argv) { fprintf(stderr, "usage: %s [-hvycdrRk] [-n ] [-f ] [-x ]\n" - " [-s ] [-p ] [-t