From b8ba93b654d9d21fd40326e5eca265eb7045c2f2 Mon Sep 17 00:00:00 2001 From: Mahadevuni Naveen Kumar Date: Tue, 24 Jun 2025 01:06:54 +0530 Subject: [PATCH] feat(function): Implement theta sketch math functions --- velox/CMakeLists.txt | 1 + velox/external/theta/BinomialBounds.h | 1032 +++ velox/external/theta/BitPacking.h | 6552 +++++++++++++++++ velox/external/theta/CMakeLists.txt | 29 + velox/external/theta/CommonDefs.h | 131 + .../theta/CompactThetaSketchParser.cpp | 261 + .../external/theta/CompactThetaSketchParser.h | 82 + velox/external/theta/ConditionalForward.h | 81 + velox/external/theta/CountZeros.h | 109 + velox/external/theta/MemoryOperations.h | 90 + velox/external/theta/MurmurHash3.h | 221 + velox/external/theta/Serde.h | 297 + velox/external/theta/ThetaComparators.h | 48 + velox/external/theta/ThetaConstants.h | 43 + velox/external/theta/ThetaHelpers.h | 83 + velox/external/theta/ThetaSketch.cpp | 1122 +++ velox/external/theta/ThetaSketch.h | 699 ++ velox/external/theta/ThetaUnion.cpp | 79 + velox/external/theta/ThetaUnion.h | 113 + velox/external/theta/ThetaUnionBase.cpp | 168 + velox/external/theta/ThetaUnionBase.h | 65 + .../external/theta/ThetaUpdateSketchBase.cpp | 530 ++ velox/external/theta/ThetaUpdateSketchBase.h | 258 + velox/external/theta/tests/BitPackingTest.cpp | 133 + velox/external/theta/tests/CMakeLists.txt | 32 + velox/external/theta/tests/TestUtils.h | 204 + .../external/theta/tests/ThetaSketchTest.cpp | 773 ++ velox/external/theta/tests/ThetaUnionTest.cpp | 164 + .../theta_compact_empty_from_java_v1.sk | Bin 0 -> 24 bytes .../theta_compact_empty_from_java_v2.sk | Bin 0 -> 8 bytes .../theta_compact_estimation_from_java_v1.sk | Bin 0 -> 34760 bytes .../theta_compact_estimation_from_java_v2.sk | Bin 0 -> 34760 bytes 32 files changed, 13400 insertions(+) create mode 100644 velox/external/theta/BinomialBounds.h create mode 100644 velox/external/theta/BitPacking.h create mode 100644 velox/external/theta/CMakeLists.txt create mode 100644 velox/external/theta/CommonDefs.h create mode 100644 velox/external/theta/CompactThetaSketchParser.cpp create mode 100644 velox/external/theta/CompactThetaSketchParser.h create mode 100644 velox/external/theta/ConditionalForward.h create mode 100644 velox/external/theta/CountZeros.h create mode 100644 velox/external/theta/MemoryOperations.h create mode 100644 velox/external/theta/MurmurHash3.h create mode 100644 velox/external/theta/Serde.h create mode 100644 velox/external/theta/ThetaComparators.h create mode 100644 velox/external/theta/ThetaConstants.h create mode 100644 velox/external/theta/ThetaHelpers.h create mode 100644 velox/external/theta/ThetaSketch.cpp create mode 100644 velox/external/theta/ThetaSketch.h create mode 100644 velox/external/theta/ThetaUnion.cpp create mode 100644 velox/external/theta/ThetaUnion.h create mode 100644 velox/external/theta/ThetaUnionBase.cpp create mode 100644 velox/external/theta/ThetaUnionBase.h create mode 100644 velox/external/theta/ThetaUpdateSketchBase.cpp create mode 100644 velox/external/theta/ThetaUpdateSketchBase.h create mode 100644 velox/external/theta/tests/BitPackingTest.cpp create mode 100644 velox/external/theta/tests/CMakeLists.txt create mode 100644 velox/external/theta/tests/TestUtils.h create mode 100644 velox/external/theta/tests/ThetaSketchTest.cpp create mode 100644 velox/external/theta/tests/ThetaUnionTest.cpp create mode 100644 velox/external/theta/tests/test_sketch_files/theta_compact_empty_from_java_v1.sk create mode 100644 velox/external/theta/tests/test_sketch_files/theta_compact_empty_from_java_v2.sk create mode 100644 velox/external/theta/tests/test_sketch_files/theta_compact_estimation_from_java_v1.sk create mode 100644 velox/external/theta/tests/test_sketch_files/theta_compact_estimation_from_java_v2.sk diff --git a/velox/CMakeLists.txt b/velox/CMakeLists.txt index 45d6a454054..b63feee4517 100644 --- a/velox/CMakeLists.txt +++ b/velox/CMakeLists.txt @@ -26,6 +26,7 @@ add_subdirectory(external/date) add_subdirectory(external/tzdb) add_subdirectory(external/md5) add_subdirectory(external/hdfs) +add_subdirectory(external/theta) # # examples depend on expression diff --git a/velox/external/theta/BinomialBounds.h b/velox/external/theta/BinomialBounds.h new file mode 100644 index 00000000000..53c561552eb --- /dev/null +++ b/velox/external/theta/BinomialBounds.h @@ -0,0 +1,1032 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include "velox/common/base/Exceptions.h" +#include +#include +#include + +/* + * This class enables the estimation of error bounds given a sample set size, + * the sampling probability theta, the number of standard deviations and a + * simple noDataSeen flag. This can be used to estimate error bounds for fixed + * threshold sampling as well as the error bounds calculations for sketches. + * + * author Alexander Saydakov + * author Lee Rhodes + * author Kevin Lang + */ + +namespace facebook::velox::common::theta { + +static constexpr double deltaOfNumStdDevs[] = { + 0.5000000000000000000, // not actually using this value + 0.1586553191586026479, + 0.0227502618904135701, + 0.0013498126861731796}; + +static constexpr double lbEquivTable[] = { + 1.0, + 2.0, + 3.0, // fake values for k = 0 + 0.78733703534118149, + 3.14426768537558132, + 13.56789685109913535, // k = 1 + 0.94091379266077979, + 2.64699271711145911, + 6.29302733018320737, // k = 2 + 0.96869128474958188, + 2.46531676590527127, + 4.97375283467403051, // k = 3 + 0.97933572521046131, + 2.37418810664669877, + 4.44899975481712318, // k = 4 + 0.98479165917274258, + 2.31863116255024693, + 4.16712379778553554, // k = 5 + 0.98806033915698777, + 2.28075536565225434, + 3.99010556144099837, // k = 6 + 0.99021896790580399, + 2.25302005857281529, + 3.86784477136922078, // k = 7 + 0.99174267079089873, + 2.23168103978522936, + 3.77784896945266269, // k = 8 + 0.99287147837287648, + 2.21465899260871879, + 3.70851932988722410, // k = 9 + 0.99373900046805375, + 2.20070155496262032, + 3.65326029076638292, // k = 10 + 0.99442519013851438, + 2.18900651202670815, + 3.60803817612955413, // k = 11 + 0.99498066823221620, + 2.17903457780744247, + 3.57024330407946877, // k = 12 + 0.99543899410224412, + 2.17040883161922693, + 3.53810982030634591, // k = 13 + 0.99582322541263579, + 2.16285726913676513, + 3.51039837124298515, // k = 14 + 0.99614973311747690, + 2.15617827879603396, + 3.48621230377099778, // k = 15 + 0.99643042892560629, + 2.15021897666090922, + 3.46488605693562590, // k = 16 + 0.99667418783778317, + 2.14486114872480016, + 3.44591466064832730, // k = 17 + 0.99688774875812669, + 2.14001181420209718, + 3.42890765690452781, // k = 18 + 0.99707632299691795, + 2.13559675336844634, + 3.41355809420343803, // k = 19 + 0.99724399084971083, + 2.13155592217421486, + 3.39962113251016262, // k = 20 + 0.99739400151915447, + 2.12784018863251845, + 3.38689892877548004, // k = 21 + 0.99752896842633731, + 2.12440890875851096, + 3.37522975271599535, // k = 22 + 0.99765101725122918, + 2.12122815311133195, + 3.36448003577621080, // k = 23 + 0.99776189496810730, + 2.11826934724291505, + 3.35453840911279144, // k = 24 + 0.99786304821586214, + 2.11550823850916458, + 3.34531123809287578, // k = 25 + 0.99795568665180667, + 2.11292409529477254, + 3.33671916527694634, // k = 26 + 0.99804083063483517, + 2.11049908609763293, + 3.32869446834217797, // k = 27 + 0.99811933910984862, + 2.10821776918189130, + 3.32117898316676019, // k = 28 + 0.99819195457286014, + 2.10606671027090897, + 3.31412243534683171, // k = 29 + 0.99825930555178388, + 2.10403415237001923, + 3.30748113008135647, // k = 30 + 0.99832193858154028, + 2.10210975877822648, + 3.30121691946897045, // k = 31 + 0.99838032666573895, + 2.10028440670842542, + 3.29529629751144171, // k = 32 + 0.99843488390555990, + 2.09855000145353188, + 3.28968974413223236, // k = 33 + 0.99848596721417948, + 2.09689934193824001, + 3.28437111460505093, // k = 34 + 0.99853390005924325, + 2.09532599155502908, + 3.27931717312372939, // k = 35 + 0.99857895741078551, + 2.09382418262592296, + 3.27450718840060517, // k = 36 + 0.99862138880970974, + 2.09238872751677718, + 3.26992261182860489, // k = 37 + 0.99866141580770318, + 2.09101494715108061, + 3.26554677962434425, // k = 38 + 0.99869923565267982, + 2.08969860402822860, + 3.26136468165239535, // k = 39 + 0.99873502010169091, + 2.08843585627218431, + 3.25736275677081721, // k = 40 + 0.99876893292508839, + 2.08722321436752623, + 3.25352872241415980, // k = 41 + 0.99880111078502409, + 2.08605749165553789, + 3.24985141664350863, // k = 42 + 0.99883168573342118, + 2.08493577529222307, + 3.24632068399498053, // k = 43 + 0.99886077231613513, + 2.08385540129560809, + 3.24292724848112357, // k = 44 + 0.99888847451828155, + 2.08281392374021834, + 3.23966263299664092, // k = 45 + 0.99891488795844907, + 2.08180908991394631, + 3.23651906111521726, // k = 46 + 0.99894010085196783, + 2.08083882998420222, + 3.23348939240611344, // k = 47 + 0.99896419358239541, + 2.07990122528650545, + 3.23056705515594444, // k = 48 + 0.99898723510594323, + 2.07899450946285924, + 3.22774598963252402, // k = 49 + 0.99900929266780736, + 2.07811704477046533, + 3.22502059972006805, // k = 50 + 0.99903043086155208, + 2.07726730587160091, + 3.22238570890294795, // k = 51 + 0.99905070073845081, + 2.07644388314946582, + 3.21983651940365689, // k = 52 + 0.99907015770423868, + 2.07564546080757850, + 3.21736857351049821, // k = 53 + 0.99908884779227947, + 2.07487081196367740, + 3.21497773796417619, // k = 54 + 0.99910681586905525, + 2.07411879634256024, + 3.21266015316183484, // k = 55 + 0.99912410177549305, + 2.07338834403498140, + 3.21041222805715165, // k = 56 + 0.99914074347179849, + 2.07267845454973099, + 3.20823061166797174, // k = 57 + 0.99915677607464204, + 2.07198819052374006, + 3.20611216970604573, // k = 58 + 0.99917223149395795, + 2.07131667846186929, + 3.20405396962596001, // k = 59 + 0.99918714153457699, + 2.07066309019154460, + 3.20205326110445299, // k = 60 + 0.99920153247185794, + 2.07002665203046377, + 3.20010746990493544, // k = 61 + 0.99921543193525508, + 2.06940663431663552, + 3.19821417453343315, // k = 62 + 0.99922886570365677, + 2.06880235245998279, + 3.19637109973109546, // k = 63 + 0.99924185357357942, + 2.06821315729285971, + 3.19457610621114441, // k = 64 + 0.99925441845175555, + 2.06763843812092318, + 3.19282717869864996, // k = 65 + 0.99926658263325407, + 2.06707761824370095, + 3.19112241228646099, // k = 66 + 0.99927836173816331, + 2.06653015295219689, + 3.18946001739936946, // k = 67 + 0.99928977431994781, + 2.06599552505539918, + 3.18783829446098821, // k = 68 + 0.99930083753795884, + 2.06547324585920933, + 3.18625564538041317, // k = 69 + 0.99931156864562354, + 2.06496285191821016, + 3.18471055124089730, // k = 70 + 0.99932197985521043, + 2.06446390392778767, + 3.18320157510865442, // k = 71 + 0.99933208559809827, + 2.06397598606787369, + 3.18172735837393361, // k = 72 + 0.99934190032416836, + 2.06349869971447220, + 3.18028661102792398, // k = 73 + 0.99935143390791836, + 2.06303166975550312, + 3.17887810481605015, // k = 74 + 0.99936070171270330, + 2.06257453607466346, + 3.17750067581857820, // k = 75 + 0.99936971103502970, + 2.06212696042919674, + 3.17615321728274580, // k = 76 + 0.99937847392385493, + 2.06168861430600714, + 3.17483467831510779, // k = 77 + 0.99938700168914352, + 2.06125918927764928, + 3.17354405480557489, // k = 78 + 0.99939530099953799, + 2.06083838987589729, + 3.17228039269048168, // k = 79 + 0.99940338278830154, + 2.06042593411496000, + 3.17104278166036124, // k = 80 + 0.99941125463777780, + 2.06002155276328835, + 3.16983035274597569, // k = 81 + 0.99941892470027938, + 2.05962498741951094, + 3.16864227952240185, // k = 82 + 0.99942640059737187, + 2.05923599161263837, + 3.16747776846497686, // k = 83 + 0.99943368842187397, + 2.05885433061945378, + 3.16633606416374391, // k = 84 + 0.99944079790603269, + 2.05847977868873500, + 3.16521644518826406, // k = 85 + 0.99944773295734990, + 2.05811212058944193, + 3.16411821883858124, // k = 86 + 0.99945450059186669, + 2.05775114781260982, + 3.16304072400711789, // k = 87 + 0.99946110646314423, + 2.05739666442039493, + 3.16198332650733960, // k = 88 + 0.99946755770463369, + 2.05704847678819647, + 3.16094541781455973, // k = 89 + 0.99947385746861528, + 2.05670640500335367, + 3.15992641851471490, // k = 90 + 0.99948001256305474, + 2.05637027420314666, + 3.15892576988736096, // k = 91 + 0.99948602689656241, + 2.05603991286400856, + 3.15794293484717059, // k = 92 + 0.99949190674294641, + 2.05571516158917689, + 3.15697740043813724, // k = 93 + 0.99949765436329585, + 2.05539586490317561, + 3.15602867309343083, // k = 94 + 0.99950327557880314, + 2.05508187237845164, + 3.15509627710042651, // k = 95 + 0.99950877461972709, + 2.05477304104951486, + 3.15417975753007340, // k = 96 + 0.99951415481862682, + 2.05446923022574879, + 3.15327867462917766, // k = 97 + 0.99951942042375208, + 2.05417030908833453, + 3.15239260700215596, // k = 98 + 0.99952457390890004, + 2.05387614661762541, + 3.15152114915238712, // k = 99 + 0.99952962005008317, + 2.05358662050909402, + 3.15066390921020911, // k = 100 + 0.99953456216121594, + 2.05330161104427589, + 3.14982051097524618, // k = 101 + 0.99953940176368405, + 2.05302100378725072, + 3.14899059183684926, // k = 102 + 0.99954414373920031, + 2.05274468493067275, + 3.14817379948561893, // k = 103 + 0.99954879047621148, + 2.05247255013657082, + 3.14736979964868624, // k = 104 + 0.99955334485656522, + 2.05220449388099269, + 3.14657826610371671, // k = 105 + 0.99955780993869325, + 2.05194041831310869, + 3.14579888316276879, // k = 106 + 0.99956218652590678, + 2.05168022402710903, + 3.14503134811607765, // k = 107 + 0.99956647932785359, + 2.05142381889103831, + 3.14427536967733090, // k = 108 + 0.99957069025060719, + 2.05117111251445294, + 3.14353066260227365, // k = 109 + 0.99957482032178291, + 2.05092201793428330, + 3.14279695558593630, // k = 110 + 0.99957887261450651, + 2.05067645094720774, + 3.14207398336887422, // k = 111 + 0.99958284988383639, + 2.05043432833224415, + 3.14136149076028914, // k = 112 + 0.99958675435604505, + 2.05019557189746138, + 3.14065923143530767, // k = 113 + 0.99959058650074439, + 2.04996010556124020, + 3.13996696426707445, // k = 114 + 0.99959434898201494, + 2.04972785368377686, + 3.13928445867830419, // k = 115 + 0.99959804437042976, + 2.04949874512311681, + 3.13861149103462367, // k = 116 + 0.99960167394553423, + 2.04927271043337100, + 3.13794784369528656, // k = 117 + 0.99960523957651048, + 2.04904968140490951, + 3.13729330661277572, // k = 118 + 0.99960874253329735, + 2.04882959397491504, + 3.13664767767019725, // k = 119 + 0.99961218434327748, + 2.04861238220240693, + 3.13601075688413289 // k = 120 +}; + +static constexpr double ubEquivTable[] = { + 1.0, + 2.0, + 3.0, // fake values for k = 0 + 0.99067760836669549, + 1.75460517119302040, + 2.48055626001627161, // k = 1 + 0.99270518097577565, + 1.78855957509907171, + 2.53863835259832626, // k = 2 + 0.99402032633599902, + 1.81047286499563143, + 2.57811676180597260, // k = 3 + 0.99492607629539975, + 1.82625928017762362, + 2.60759550546498531, // k = 4 + 0.99558653966013821, + 1.83839160339161367, + 2.63086812358551470, // k = 5 + 0.99608981951632813, + 1.84812399034444752, + 2.64993712523727254, // k = 6 + 0.99648648035983456, + 1.85617372053235385, + 2.66598485907860550, // k = 7 + 0.99680750790483330, + 1.86298655802610824, + 2.67976541374471822, // k = 8 + 0.99707292880049181, + 1.86885682585270274, + 2.69178781407745760, // k = 9 + 0.99729614928489241, + 1.87398826101983218, + 2.70241106542158604, // k = 10 + 0.99748667952445658, + 1.87852708449801753, + 2.71189717290596377, // k = 11 + 0.99765127712748836, + 1.88258159501103250, + 2.72044290303773550, // k = 12 + 0.99779498340305395, + 1.88623391878036273, + 2.72819957382063194, // k = 13 + 0.99792160418357412, + 1.88954778748873764, + 2.73528576807902368, // k = 14 + 0.99803398604944960, + 1.89257337682371940, + 2.74179612106766513, // k = 15 + 0.99813449883217231, + 1.89535099316557876, + 2.74780718300419835, // k = 16 + 0.99822494122659577, + 1.89791339232732525, + 2.75338173141955167, // k = 17 + 0.99830679915913834, + 1.90028752122407241, + 2.75857186416826039, // k = 18 + 0.99838117410831728, + 1.90249575897183831, + 2.76342117562634826, // k = 19 + 0.99844913407071090, + 1.90455689090418900, + 2.76796659454200267, // k = 20 + 0.99851147736424650, + 1.90648682834171268, + 2.77223944710058845, // k = 21 + 0.99856879856019987, + 1.90829917277082473, + 2.77626682032629901, // k = 22 + 0.99862183849734265, + 1.91000561415842185, + 2.78007199816156003, // k = 23 + 0.99867096266018507, + 1.91161621560812023, + 2.78367524259661536, // k = 24 + 0.99871656986212543, + 1.91313978579765376, + 2.78709435016625662, // k = 25 + 0.99875907577771272, + 1.91458400425526065, + 2.79034488416175463, // k = 26 + 0.99879885565047744, + 1.91595563175945927, + 2.79344064132371273, // k = 27 + 0.99883610756373287, + 1.91726064301425936, + 2.79639384757751941, // k = 28 + 0.99887095169674467, + 1.91850441099725799, + 2.79921543574803877, // k = 29 + 0.99890379414739527, + 1.91969155477030995, + 2.80191513182441554, // k = 30 + 0.99893466279047516, + 1.92082633358913313, + 2.80450167352080371, // k = 31 + 0.99896392088177777, + 1.92191254955568525, + 2.80698295731653502, // k = 32 + 0.99899147889385631, + 1.92295362479495680, + 2.80936614404217266, // k = 33 + 0.99901764688726757, + 1.92395267400968351, + 2.81165765979318394, // k = 34 + 0.99904238606342233, + 1.92491244978191389, + 2.81386337393604435, // k = 35 + 0.99906590152386343, + 1.92583552644848055, + 2.81598868034527072, // k = 36 + 0.99908829040739988, + 1.92672418013918900, + 2.81803841726804194, // k = 37 + 0.99910959420023460, + 1.92758051694144683, + 2.82001709302821268, // k = 38 + 0.99912996403594434, + 1.92840654943159961, + 2.82192875763732332, // k = 39 + 0.99914930224576892, + 1.92920397044028391, + 2.82377730628954282, // k = 40 + 0.99916781270195543, + 1.92997447498220254, + 2.82556612075063640, // k = 41 + 0.99918553179077207, + 1.93071949211818605, + 2.82729843191989971, // k = 42 + 0.99920250730914972, + 1.93144048613876862, + 2.82897728689417249, // k = 43 + 0.99921873345181211, + 1.93213870990595638, + 2.83060537017752267, // k = 44 + 0.99923435180002684, + 1.93281536508689555, + 2.83218527795750674, // k = 45 + 0.99924930425362390, + 1.93347145882316340, + 2.83371938965598247, // k = 46 + 0.99926370394567243, + 1.93410820221384938, + 2.83520990872793277, // k = 47 + 0.99927750755296074, + 1.93472643138986200, + 2.83665891945119597, // k = 48 + 0.99929082941537217, + 1.93532697329771963, + 2.83806833931606661, // k = 49 + 0.99930366295501472, + 1.93591074716263734, + 2.83943997143404658, // k = 50 + 0.99931598804721489, + 1.93647857274021362, + 2.84077557836653227, // k = 51 + 0.99932789059798210, + 1.93703110239354714, + 2.84207662106302905, // k = 52 + 0.99933946180485123, + 1.93756904936378760, + 2.84334468086129277, // k = 53 + 0.99935053819703512, + 1.93809302131219852, + 2.84458116874117195, // k = 54 + 0.99936126637970801, + 1.93860365411038060, + 2.84578731838604426, // k = 55 + 0.99937166229284458, + 1.93910149816429112, + 2.84696443486512862, // k = 56 + 0.99938169190727422, + 1.93958709548454067, + 2.84811369085281285, // k = 57 + 0.99939136927613959, + 1.94006085573701625, + 2.84923617230361970, // k = 58 + 0.99940074328745254, + 1.94052339623206649, + 2.85033291216254270, // k = 59 + 0.99940993070470086, + 1.94097508636855309, + 2.85140492437699322, // k = 60 + 0.99941868577388959, + 1.94141633372043998, + 2.85245314430358121, // k = 61 + 0.99942734443487780, + 1.94184757038001976, + 2.85347839582286156, // k = 62 + 0.99943556385736088, + 1.94226915100517772, + 2.85448160365493209, // k = 63 + 0.99944374522542034, + 1.94268143723749631, + 2.85546346373061510, // k = 64 + 0.99945159955424856, + 1.94308482059116727, + 2.85642486111805738, // k = 65 + 0.99945915301904620, + 1.94347956957849988, + 2.85736639994965458, // k = 66 + 0.99946660663832176, + 1.94386600964031686, + 2.85828887832701639, // k = 67 + 0.99947383703224091, + 1.94424436597356021, + 2.85919278275500233, // k = 68 + 0.99948075442870277, + 1.94461502153473020, + 2.86007887186090670, // k = 69 + 0.99948766082269458, + 1.94497821937304138, + 2.86094774077355396, // k = 70 + 0.99949422748713346, + 1.94533411296001191, + 2.86179981848076181, // k = 71 + 0.99950070756119658, + 1.94568300035135167, + 2.86263579405672886, // k = 72 + 0.99950704321753392, + 1.94602523449961495, + 2.86345610449197352, // k = 73 + 0.99951320334216121, + 1.94636083782822311, + 2.86426125541271404, // k = 74 + 0.99951920293474927, + 1.94669011080745236, + 2.86505169255406145, // k = 75 + 0.99952501670378524, + 1.94701327348536779, + 2.86582788270862920, // k = 76 + 0.99953071209267819, + 1.94733044372333097, + 2.86659027602854621, // k = 77 + 0.99953632734991515, + 1.94764180764266825, + 2.86733927778843167, // k = 78 + 0.99954171164873173, + 1.94794766430732125, + 2.86807526143834934, // k = 79 + 0.99954699274462655, + 1.94824807472994621, + 2.86879864789403882, // k = 80 + 0.99955216611081710, + 1.94854317889829076, + 2.86950970901679625, // k = 81 + 0.99955730019613043, + 1.94883320227168610, + 2.87020887436986527, // k = 82 + 0.99956213770650493, + 1.94911826561721568, + 2.87089648477021342, // k = 83 + 0.99956704264963037, + 1.94939848545763539, + 2.87157281693902178, // k = 84 + 0.99957166306481327, + 1.94967401618316671, + 2.87223821840905202, // k = 85 + 0.99957632713136491, + 1.94994497791333288, + 2.87289293193450135, // k = 86 + 0.99958087233392234, + 1.95021155752212394, + 2.87353731228213860, // k = 87 + 0.99958532555996271, + 1.95047376805584349, + 2.87417154907075201, // k = 88 + 0.99958956246481989, + 1.95073180380688882, + 2.87479599765507032, // k = 89 + 0.99959389351869277, + 1.95098572880579013, + 2.87541081987382086, // k = 90 + 0.99959807862052230, + 1.95123574036898617, + 2.87601637401948551, // k = 91 + 0.99960214057801977, + 1.95148186921983324, + 2.87661283691068093, // k = 92 + 0.99960607527256684, + 1.95172415829728152, + 2.87720042968334155, // k = 93 + 0.99960996433179616, + 1.95196280898670693, + 2.87777936649376898, // k = 94 + 0.99961379137860717, + 1.95219787713926962, + 2.87834989933620022, // k = 95 + 0.99961756088146103, + 1.95242944583677058, + 2.87891216133900230, // k = 96 + 0.99962125605327401, + 1.95265762420910960, + 2.87946647367488140, // k = 97 + 0.99962486179100551, + 1.95288245314810638, + 2.88001290210658567, // k = 98 + 0.99962843240297161, + 1.95310404286672679, + 2.88055166523392359, // k = 99 + 0.99963187276145504, + 1.95332251980147475, + 2.88108300006589957, // k = 100 + 0.99963525453173929, + 1.95353785898848287, + 2.88160703591438505, // k = 101 + 0.99963855412988778, + 1.95375019354571577, + 2.88212393551896184, // k = 102 + 0.99964190254169694, + 1.95395953472205974, + 2.88263389761985422, // k = 103 + 0.99964506565942202, + 1.95416607430155409, + 2.88313700661564098, // k = 104 + 0.99964834424233118, + 1.95436972855640079, + 2.88363350163803034, // k = 105 + 0.99965136548857458, + 1.95457068540693513, + 2.88412349413960101, // k = 106 + 0.99965436594726498, + 1.95476896383092935, + 2.88460710620208260, // k = 107 + 0.99965736463468602, + 1.95496457504532373, + 2.88508450078833789, // k = 108 + 0.99966034130443404, + 1.95515761150707590, + 2.88555580586194083, // k = 109 + 0.99966326130828520, + 1.95534810382198998, + 2.88602118761679094, // k = 110 + 0.99966601446035952, + 1.95553622237747504, + 2.88648066384146773, // k = 111 + 0.99966887679593697, + 1.95572186728168163, + 2.88693444915907094, // k = 112 + 0.99967161286551232, + 1.95590523410490391, + 2.88738271495714116, // k = 113 + 0.99967435412270333, + 1.95608626483223702, + 2.88782540459769166, // k = 114 + 0.99967701261934394, + 1.95626497627117146, + 2.88826277189363623, // k = 115 + 0.99967963265157778, + 1.95644153684824573, + 2.88869486674335008, // k = 116 + 0.99968216317182623, + 1.95661589936000269, + 2.88912184353694101, // k = 117 + 0.99968479674396349, + 1.95678821614791332, + 2.88954376359643561, // k = 118 + 0.99968729031337489, + 1.95695842061650183, + 2.88996069422501023, // k = 119 + 0.99968963358631413, + 1.95712651709766305, + 2.89037285320668502 // k = 120 +}; + +class BinomialBounds { + public: + static double getLowerBound( + unsigned long long numSamples, + double theta, + unsigned numStdDevs) { + checkTheta(theta); + checkNumStdDevs(numStdDevs); + const double estimate = numSamples / theta; + const double lb = + computeApproxBinomialLowerBound(numSamples, theta, numStdDevs); + return std::min(estimate, std::max(static_cast(numSamples), lb)); + } + + static double getUpperBound( + unsigned long long numSamples, + double theta, + unsigned numStdDevs) { + checkTheta(theta); + checkNumStdDevs(numStdDevs); + const double estimate = numSamples / theta; + const double ub = + computeApproxBinomialUpperBound(numSamples, theta, numStdDevs); + return std::max(estimate, ub); + } + + private: + // our "classic" bounds, but now with continuity correction + static double contClassicLb( + unsigned long long numSamples, + double theta, + double numStdDevs) { + const double n_hat = (numSamples - 0.5) / theta; + const double b = numStdDevs * std::sqrt((1.0 - theta) / theta); + const double d = 0.5 * b * std::sqrt((b * b) + (4.0 * n_hat)); + const double center = n_hat + (0.5 * (b * b)); + return (center - d); + } + + // our "classic" bounds, but now with continuity correction + static double contClassicUb( + unsigned long long numSamples, + double theta, + double numStdSevs) { + const double n_hat = (numSamples + 0.5) / theta; + const double b = numStdSevs * std::sqrt((1.0 - theta) / theta); + const double d = 0.5 * b * std::sqrt((b * b) + (4.0 * n_hat)); + const double center = n_hat + (0.5 * (b * b)); + return (center + d); + } + + // This is a special purpose calculator for NStar, using a computational + // strategy inspired by its Bayesian definition. It is only appropriate + // for a very limited set of inputs. However, the procedure + // compute_approx_binomial_lower_bound() below does in fact only call it for + // suitably limited inputs. Outside of this limited range, two different bad + // things will happen. First, because we are not using logarithms, the values + // of intermediate quantities will exceed the dynamic range of doubles. + // Second, even if that problem were fixed, the running time of this procedure + // is essentially linear in est = (numSamples / p), and that can be Very, Very + // Big. + static unsigned long long + specialNStar(unsigned long long numSamples, double p, double delta) { + const double q = 1.0 - p; + // Use a different algorithm if the following is true; this one will be too + // slow, or worse. + if ((numSamples / p) >= 500.0) + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "out of range", + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); + double cur_term = + std::pow(p, numSamples); // curTerm = posteriorProbability (k, k, p) + if (cur_term <= 1e-100) + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "out of range", + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); // sanity check for non-use of logarithms + double tot = cur_term; + unsigned long long m = numSamples; + while (tot <= delta) { // this test can fail even the first time + cur_term = (cur_term * q * (m)) / ((m + 1) - numSamples); + tot += cur_term; + m += 1; + } + // we have reached a state where tot > delta, so back up one + return (m - 1); + } + + // The following procedure has very limited applicability. + // The above remarks about special_n_star() also apply here. + static unsigned long long + specialNPrimeB(unsigned long long numSamples, double p, double delta) { + const double q = 1.0 - p; + const double oneMinusDelta = 1.0 - delta; + double curTerm = + std::pow(p, numSamples); // curTerm = posteriorProbability (k, k, p) + if (curTerm <= 1e-100) + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "out of range", + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); // sanity check for non-use of logarithms + double tot = curTerm; + unsigned long long m = numSamples; + while (tot < oneMinusDelta) { + curTerm = (curTerm * q * (m)) / ((m + 1) - numSamples); + tot += curTerm; + m += 1; + } + return m; // no need to back up + } + + static unsigned long long + specialNPrimeF(unsigned long long numSamples, double p, double delta) { + // Use a different algorithm if the following is true; this one will be too + // slow, or worse. + if ((numSamples / p) >= 500.0) + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "out of range", + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); // A super-small delta could also make it slow. + return specialNPrimeB(numSamples + 1, p, delta); + } + + // The following computes an approximation to the lower bound of a Frequentist + // confidence interval based on the tails of the Binomial distribution. + static double computeApproxBinomialLowerBound( + unsigned long long numSamples, + double theta, + unsigned numStdDevs) { + if (theta == 1) + return static_cast(numSamples); + if (numSamples == 0) + return 0; + if (numSamples == 1) { + const double delta = deltaOfNumStdDevs[numStdDevs]; + const double rawLb = std::log(1 - delta) / std::log(1 - theta); + return std::floor(rawLb); // round down + } + if (numSamples > 120) { + // plenty of samples, so gaussian approximation to binomial distribution + // isn't too bad + const double rawLb = contClassicLb(numSamples, theta, numStdDevs); + return (rawLb - 0.5); // fake round down + } + // at this point we know 2 <= num_samples <= 120 + if (theta > (1 - 1e-5)) { // empirically-determined threshold + return static_cast(numSamples); + } + if (theta < (numSamples / 360.0)) { // empirically-determined threshold + // here we use the Gaussian approximation, but with a modified + // num_std_devs + const unsigned index = + 3 * static_cast(numSamples) + (numStdDevs - 1); + const double rawLb = + contClassicLb(numSamples, theta, lbEquivTable[index]); + return rawLb - 0.5; // fake round down + } + // This is the most difficult range to approximate; we will compute an + // "exact" LB. We know that est <= 360, so specialNStar() shouldn't be + // ridiculously slow. + const double delta = deltaOfNumStdDevs[numStdDevs]; + return static_cast( + specialNStar(numSamples, theta, delta)); // no need to round + } + + // The following computes an approximation to the upper bound of a Frequentist + // confidence interval based on the tails of the Binomial distribution. + static double computeApproxBinomialUpperBound( + unsigned long long numSamples, + double theta, + unsigned numStdDevs) { + if (theta == 1) + return static_cast(numSamples); + if (numSamples == 0) { + const double delta = deltaOfNumStdDevs[numStdDevs]; + const double raw_ub = std::log(delta) / std::log(1 - theta); + return std::ceil(raw_ub); // round up + } + if (numSamples > 120) { + // plenty of samples, so gaussian approximation to binomial distribution + // isn't too bad + const double raw_ub = contClassicUb(numSamples, theta, numStdDevs); + return (raw_ub + 0.5); // fake round up + } + // at this point we know 2 <= num_samples <= 120 + if (theta > (1 - 1e-5)) { // empirically-determined threshold + return static_cast(numSamples + 1); + } + if (theta < (numSamples / 360.0)) { // empirically-determined threshold + // here we use the Gaussian approximation, but with a modified + // num_std_devs + const unsigned index = + 3 * static_cast(numSamples) + (numStdDevs - 1); + const double raw_ub = + contClassicUb(numSamples, theta, ubEquivTable[index]); + return raw_ub + 0.5; // fake round up + } + // This is the most difficult range to approximate; we will compute an + // "exact" UB. We know that est <= 360, so specialNPrimeF() shouldn't be + // ridiculously slow. + const double delta = deltaOfNumStdDevs[numStdDevs]; + return static_cast( + specialNPrimeF(numSamples, theta, delta)); // no need to round + } + + static void checkTheta(double theta) { + if (theta < 0 || theta > 1) { + throw VeloxUserError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "theta must be in [0, 1]", + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); + } + } + + static void checkNumStdDevs(unsigned numStdDevs) { + if (numStdDevs < 1 || numStdDevs > 3) { + throw VeloxUserError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "num_std_devs must be 1, 2 or 3", + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); + } + } +}; + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/BitPacking.h b/velox/external/theta/BitPacking.h new file mode 100644 index 00000000000..e964e1316a8 --- /dev/null +++ b/velox/external/theta/BitPacking.h @@ -0,0 +1,6552 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include "velox/common/base/Exceptions.h" + +#include +#include + +namespace facebook::velox::common::theta { + +static inline uint8_t +packBits(uint64_t value, uint8_t bits, uint8_t*& ptr, uint8_t offset) { + if (offset > 0) { + const uint8_t chunkBits = 8 - offset; + const uint8_t mask = (1 << chunkBits) - 1; + if (bits < chunkBits) { + *ptr |= (value << (chunkBits - bits)) & mask; + return offset + bits; + } + *ptr++ |= (value >> (bits - chunkBits)) & mask; + bits -= chunkBits; + } + while (bits >= 8) { + *ptr++ = static_cast(value >> (bits - 8)); + bits -= 8; + } + if (bits > 0) { + *ptr = static_cast(value << (8 - bits)); + return bits; + } + return 0; +} + +static inline uint8_t +unpackBits(uint64_t& value, uint8_t bits, const uint8_t*& ptr, uint8_t offset) { + const uint8_t availBits = 8 - offset; + const uint8_t chunkBits = std::min(availBits, bits); + const uint8_t mask = (1 << chunkBits) - 1; + value = (*ptr >> (availBits - chunkBits)) & mask; + ptr += availBits == chunkBits; + offset = (offset + chunkBits) & 7; + bits -= chunkBits; + while (bits >= 8) { + value <<= 8; + value |= *ptr++; + bits -= 8; + } + if (bits > 0) { + value <<= bits; + value |= *ptr >> (8 - bits); + return bits; + } + return offset; +} + +// pack given number of bits from a block of 8 64-bit values into bytes +// we don't need 0 and 64 bits +// we assume that higher bits (which we are not packing) are zeros +// this assumption allows to avoid masking operations + +static inline void packBits1(const uint64_t* values, uint8_t* ptr) { + *ptr = static_cast(values[0] << 7); + *ptr |= static_cast(values[1] << 6); + *ptr |= static_cast(values[2] << 5); + *ptr |= static_cast(values[3] << 4); + *ptr |= static_cast(values[4] << 3); + *ptr |= static_cast(values[5] << 2); + *ptr |= static_cast(values[6] << 1); + *ptr |= static_cast(values[7]); +} + +static inline void packBits2(const uint64_t* values, uint8_t* ptr) { + *ptr = static_cast(values[0] << 6); + *ptr |= static_cast(values[1] << 4); + *ptr |= static_cast(values[2] << 2); + *ptr++ |= static_cast(values[3]); + + *ptr = static_cast(values[4] << 6); + *ptr |= static_cast(values[5] << 4); + *ptr |= static_cast(values[6] << 2); + *ptr |= static_cast(values[7]); +} + +static inline void packBits3(const uint64_t* values, uint8_t* ptr) { + *ptr = static_cast(values[0] << 5); + *ptr |= static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 1); + + *ptr = static_cast(values[2] << 7); + *ptr |= static_cast(values[3] << 4); + *ptr |= static_cast(values[4] << 1); + *ptr++ |= static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr |= static_cast(values[6] << 3); + *ptr |= static_cast(values[7]); +} + +static inline void packBits4(const uint64_t* values, uint8_t* ptr) { + *ptr = static_cast(values[0] << 4); + *ptr++ |= static_cast(values[1]); + + *ptr = static_cast(values[2] << 4); + *ptr++ |= static_cast(values[3]); + + *ptr = static_cast(values[4] << 4); + *ptr++ |= static_cast(values[5]); + + *ptr = static_cast(values[6] << 4); + *ptr |= static_cast(values[7]); +} + +static inline void packBits5(const uint64_t* values, uint8_t* ptr) { + *ptr = static_cast(values[0] << 3); + *ptr++ |= static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr |= static_cast(values[2] << 1); + *ptr++ |= static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 1); + + *ptr = static_cast(values[4] << 7); + *ptr |= static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 3); + + *ptr = static_cast(values[6] << 5); + *ptr |= static_cast(values[7]); +} + +static inline void packBits6(const uint64_t* values, uint8_t* ptr) { + *ptr = static_cast(values[0] << 2); + *ptr++ |= static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 2); + + *ptr = static_cast(values[2] << 6); + *ptr++ |= static_cast(values[3]); + + *ptr = static_cast(values[4] << 2); + *ptr++ |= static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 2); + + *ptr = static_cast(values[6] << 6); + *ptr |= static_cast(values[7]); +} + +static inline void packBits7(const uint64_t* values, uint8_t* ptr) { + *ptr = static_cast(values[0] << 1); + *ptr++ |= static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 5); + + *ptr = static_cast(values[2] << 3); + *ptr++ |= static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 3); + + *ptr = static_cast(values[4] << 5); + *ptr++ |= static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 1); + + *ptr = static_cast(values[6] << 7); + *ptr |= static_cast(values[7]); +} + +static inline void packBits8(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0]); + *ptr++ = static_cast(values[1]); + *ptr++ = static_cast(values[2]); + *ptr++ = static_cast(values[3]); + *ptr++ = static_cast(values[4]); + *ptr++ = static_cast(values[5]); + *ptr++ = static_cast(values[6]); + *ptr = static_cast(values[7]); +} + +static inline void packBits9(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 1); + + *ptr = static_cast(values[0] << 7); + *ptr++ |= static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 3); + + *ptr = static_cast(values[2] << 5); + *ptr++ |= static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 5); + + *ptr = static_cast(values[4] << 3); + *ptr++ |= static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 7); + + *ptr = static_cast(values[6] << 1); + *ptr++ |= static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits10(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 2); + + *ptr = static_cast(values[0] << 6); + *ptr++ |= static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 6); + + *ptr = static_cast(values[2] << 2); + *ptr++ |= static_cast(values[3] >> 8); + + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 2); + + *ptr = static_cast(values[4] << 6); + *ptr++ |= static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 6); + + *ptr = static_cast(values[6] << 2); + *ptr++ |= static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits11(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 3); + + *ptr = static_cast(values[0] << 5); + *ptr++ |= static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 9); + + *ptr++ = static_cast(values[2] >> 1); + + *ptr = static_cast(values[2] << 7); + *ptr++ |= static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 7); + + *ptr = static_cast(values[4] << 1); + *ptr++ |= static_cast(values[5] >> 10); + + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 5); + + *ptr = static_cast(values[6] << 3); + *ptr++ |= static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits12(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 4); + + *ptr = static_cast(values[0] << 4); + *ptr++ |= static_cast(values[1] >> 8); + + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 4); + + *ptr = static_cast(values[2] << 4); + *ptr++ |= static_cast(values[3] >> 8); + + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 4); + + *ptr = static_cast(values[4] << 4); + *ptr++ |= static_cast(values[5] >> 8); + + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 4); + + *ptr = static_cast(values[6] << 4); + *ptr++ |= static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits13(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 5); + + *ptr = static_cast(values[0] << 3); + *ptr++ |= static_cast(values[1] >> 10); + + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 7); + + *ptr = static_cast(values[2] << 1); + *ptr++ |= static_cast(values[3] >> 12); + + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 9); + + *ptr++ = static_cast(values[4] >> 1); + + *ptr = static_cast(values[4] << 7); + *ptr++ |= static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 11); + + *ptr++ = static_cast(values[6] >> 3); + + *ptr = static_cast(values[6] << 5); + *ptr++ |= static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits14(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 6); + + *ptr = static_cast(values[0] << 2); + *ptr++ |= static_cast(values[1] >> 12); + + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 10); + + *ptr++ = static_cast(values[2] >> 2); + + *ptr = static_cast(values[2] << 6); + *ptr++ |= static_cast(values[3] >> 8); + + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 6); + + *ptr = static_cast(values[4] << 2); + *ptr++ |= static_cast(values[5] >> 12); + + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 10); + + *ptr++ = static_cast(values[6] >> 2); + + *ptr = static_cast(values[6] << 6); + *ptr++ |= static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits15(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 7); + + *ptr = static_cast(values[0] << 1); + *ptr++ |= static_cast(values[1] >> 14); + + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 13); + + *ptr++ = static_cast(values[2] >> 5); + + *ptr = static_cast(values[2] << 3); + *ptr++ |= static_cast(values[3] >> 12); + + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 11); + + *ptr++ = static_cast(values[4] >> 3); + + *ptr = static_cast(values[4] << 5); + *ptr++ |= static_cast(values[5] >> 10); + + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 9); + + *ptr++ = static_cast(values[6] >> 1); + + *ptr = static_cast(values[6] << 7); + *ptr++ |= static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits16(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 8); + *ptr++ = static_cast(values[0]); + + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 8); + *ptr++ = static_cast(values[2]); + + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 8); + *ptr++ = static_cast(values[4]); + + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 8); + *ptr++ = static_cast(values[6]); + + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits17(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 9); + + *ptr++ = static_cast(values[0] >> 1); + + *ptr = static_cast(values[0] << 7); + *ptr++ |= static_cast(values[1] >> 10); + + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 11); + + *ptr++ = static_cast(values[2] >> 3); + + *ptr = static_cast(values[2] << 5); + *ptr++ |= static_cast(values[3] >> 12); + + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 13); + + *ptr++ = static_cast(values[4] >> 5); + + *ptr = static_cast(values[4] << 3); + *ptr++ |= static_cast(values[5] >> 14); + + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 15); + + *ptr++ = static_cast(values[6] >> 7); + + *ptr = static_cast(values[6] << 1); + *ptr++ |= static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits18(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 10); + + *ptr++ = static_cast(values[0] >> 2); + + *ptr = static_cast(values[0] << 6); + *ptr++ |= static_cast(values[1] >> 12); + + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 14); + + *ptr++ = static_cast(values[2] >> 6); + + *ptr = static_cast(values[2] << 2); + *ptr++ |= static_cast(values[3] >> 16); + + *ptr++ = static_cast(values[3] >> 8); + + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 10); + + *ptr++ = static_cast(values[4] >> 2); + + *ptr = static_cast(values[4] << 6); + *ptr++ |= static_cast(values[5] >> 12); + + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 14); + + *ptr++ = static_cast(values[6] >> 6); + + *ptr = static_cast(values[6] << 2); + *ptr++ |= static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits19(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 11); + + *ptr++ = static_cast(values[0] >> 3); + + *ptr = static_cast(values[0] << 5); + *ptr++ |= static_cast(values[1] >> 14); + + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 17); + + *ptr++ = static_cast(values[2] >> 9); + + *ptr++ = static_cast(values[2] >> 1); + + *ptr = static_cast(values[2] << 7); + *ptr++ |= static_cast(values[3] >> 12); + + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 15); + + *ptr++ |= static_cast(values[4] >> 7); + + *ptr = static_cast(values[4] << 1); + *ptr++ |= static_cast(values[5] >> 18); + + *ptr++ = static_cast(values[5] >> 10); + + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 13); + + *ptr++ = static_cast(values[6] >> 5); + + *ptr = static_cast(values[6] << 3); + *ptr++ |= static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits20(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 12); + + *ptr++ = static_cast(values[0] >> 4); + + *ptr = static_cast(values[0] << 4); + *ptr++ |= static_cast(values[1] >> 16); + + *ptr++ = static_cast(values[1] >> 8); + + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 12); + + *ptr++ = static_cast(values[2] >> 4); + + *ptr = static_cast(values[2] << 4); + *ptr++ |= static_cast(values[3] >> 16); + + *ptr++ = static_cast(values[3] >> 8); + + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 12); + + *ptr++ = static_cast(values[4] >> 4); + + *ptr = static_cast(values[4] << 4); + *ptr++ |= static_cast(values[5] >> 16); + + *ptr++ = static_cast(values[5] >> 8); + + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 12); + + *ptr++ = static_cast(values[6] >> 4); + + *ptr = static_cast(values[6] << 4); + *ptr++ |= static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits21(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 13); + + *ptr++ = static_cast(values[0] >> 5); + + *ptr = static_cast(values[0] << 3); + *ptr++ |= static_cast(values[1] >> 18); + + *ptr++ = static_cast(values[1] >> 10); + + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 15); + + *ptr++ = static_cast(values[2] >> 7); + + *ptr = static_cast(values[2] << 1); + *ptr++ |= static_cast(values[3] >> 20); + + *ptr++ = static_cast(values[3] >> 12); + + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 17); + + *ptr++ = static_cast(values[4] >> 9); + + *ptr++ = static_cast(values[4] >> 1); + + *ptr = static_cast(values[4] << 7); + *ptr++ |= static_cast(values[5] >> 14); + + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 19); + + *ptr++ = static_cast(values[6] >> 11); + + *ptr++ = static_cast(values[6] >> 3); + + *ptr = static_cast(values[6] << 5); + *ptr++ |= static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits22(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 14); + + *ptr++ = static_cast(values[0] >> 6); + + *ptr = static_cast(values[0] << 2); + *ptr++ |= static_cast(values[1] >> 20); + + *ptr++ = static_cast(values[1] >> 12); + + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 18); + + *ptr++ = static_cast(values[2] >> 10); + + *ptr++ = static_cast(values[2] >> 2); + + *ptr = static_cast(values[2] << 6); + *ptr++ |= static_cast(values[3] >> 16); + + *ptr++ = static_cast(values[3] >> 8); + + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 14); + + *ptr++ = static_cast(values[4] >> 6); + + *ptr = static_cast(values[4] << 2); + *ptr++ |= static_cast(values[5] >> 20); + + *ptr++ = static_cast(values[5] >> 12); + + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 18); + + *ptr++ = static_cast(values[6] >> 10); + + *ptr++ = static_cast(values[6] >> 2); + + *ptr = static_cast(values[6] << 6); + *ptr++ |= static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits23(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 15); + + *ptr++ = static_cast(values[0] >> 7); + + *ptr = static_cast(values[0] << 1); + *ptr++ |= static_cast(values[1] >> 22); + + *ptr++ = static_cast(values[1] >> 14); + + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 21); + + *ptr++ = static_cast(values[2] >> 13); + + *ptr++ = static_cast(values[2] >> 5); + + *ptr = static_cast(values[2] << 3); + *ptr++ |= static_cast(values[3] >> 20); + + *ptr++ = static_cast(values[3] >> 12); + + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 19); + + *ptr++ = static_cast(values[4] >> 11); + + *ptr++ = static_cast(values[4] >> 3); + + *ptr = static_cast(values[4] << 5); + *ptr++ |= static_cast(values[5] >> 18); + + *ptr++ = static_cast(values[5] >> 10); + + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 17); + + *ptr++ = static_cast(values[6] >> 9); + + *ptr++ = static_cast(values[6] >> 1); + + *ptr = static_cast(values[6] << 7); + *ptr++ |= static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits24(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 16); + *ptr++ = static_cast(values[0] >> 8); + *ptr++ = static_cast(values[0]); + + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 16); + *ptr++ = static_cast(values[2] >> 8); + *ptr++ = static_cast(values[2]); + + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 16); + *ptr++ = static_cast(values[4] >> 8); + *ptr++ = static_cast(values[4]); + + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 16); + *ptr++ = static_cast(values[6] >> 8); + *ptr++ = static_cast(values[6]); + + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits25(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 17); + + *ptr++ = static_cast(values[0] >> 9); + + *ptr++ = static_cast(values[0] >> 1); + + *ptr = static_cast(values[0] << 7); + *ptr++ |= static_cast(values[1] >> 18); + + *ptr++ = static_cast(values[1] >> 10); + + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 19); + + *ptr++ = static_cast(values[2] >> 11); + + *ptr++ = static_cast(values[2] >> 3); + + *ptr = static_cast(values[2] << 5); + *ptr++ |= static_cast(values[3] >> 20); + + *ptr++ = static_cast(values[3] >> 12); + + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 21); + + *ptr++ = static_cast(values[4] >> 13); + + *ptr++ = static_cast(values[4] >> 5); + + *ptr = static_cast(values[4] << 3); + *ptr++ |= static_cast(values[5] >> 22); + + *ptr++ = static_cast(values[5] >> 14); + + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 23); + + *ptr++ = static_cast(values[6] >> 15); + + *ptr++ = static_cast(values[6] >> 7); + + *ptr = static_cast(values[6] << 1); + *ptr++ |= static_cast(values[7] >> 24); + + *ptr++ = static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits26(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 18); + + *ptr++ = static_cast(values[0] >> 10); + + *ptr++ = static_cast(values[0] >> 2); + + *ptr = static_cast(values[0] << 6); + *ptr++ |= static_cast(values[1] >> 20); + + *ptr++ = static_cast(values[1] >> 12); + + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 22); + + *ptr++ = static_cast(values[2] >> 14); + + *ptr++ = static_cast(values[2] >> 6); + + *ptr = static_cast(values[2] << 2); + *ptr++ |= static_cast(values[3] >> 24); + + *ptr++ = static_cast(values[3] >> 16); + + *ptr++ = static_cast(values[3] >> 8); + + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 18); + + *ptr++ = static_cast(values[4] >> 10); + + *ptr++ = static_cast(values[4] >> 2); + + *ptr = static_cast(values[4] << 6); + *ptr++ |= static_cast(values[5] >> 20); + + *ptr++ = static_cast(values[5] >> 12); + + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 22); + + *ptr++ = static_cast(values[6] >> 14); + + *ptr++ = static_cast(values[6] >> 6); + + *ptr = static_cast(values[6] << 2); + *ptr++ |= static_cast(values[7] >> 24); + + *ptr++ = static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits27(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 19); + + *ptr++ = static_cast(values[0] >> 11); + + *ptr++ = static_cast(values[0] >> 3); + + *ptr = static_cast(values[0] << 5); + *ptr++ |= static_cast(values[1] >> 22); + + *ptr++ = static_cast(values[1] >> 14); + + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 25); + + *ptr++ = static_cast(values[2] >> 17); + + *ptr++ = static_cast(values[2] >> 9); + + *ptr++ = static_cast(values[2] >> 1); + + *ptr = static_cast(values[2] << 7); + *ptr++ |= static_cast(values[3] >> 20); + + *ptr++ = static_cast(values[3] >> 12); + + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 23); + + *ptr++ = static_cast(values[4] >> 15); + + *ptr++ = static_cast(values[4] >> 7); + + *ptr = static_cast(values[4] << 1); + *ptr++ |= static_cast(values[5] >> 26); + + *ptr++ = static_cast(values[5] >> 18); + + *ptr++ = static_cast(values[5] >> 10); + + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 21); + + *ptr++ = static_cast(values[6] >> 13); + + *ptr++ = static_cast(values[6] >> 5); + + *ptr = static_cast(values[6] << 3); + *ptr++ |= static_cast(values[7] >> 24); + + *ptr++ = static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits28(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 20); + *ptr++ = static_cast(values[0] >> 12); + *ptr++ = static_cast(values[0] >> 4); + *ptr = static_cast(values[0] << 4); + *ptr++ |= static_cast(values[1] >> 24); + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + *ptr++ = static_cast(values[2] >> 20); + *ptr++ = static_cast(values[2] >> 12); + *ptr++ = static_cast(values[2] >> 4); + *ptr = static_cast(values[2] << 4); + *ptr++ |= static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + *ptr++ = static_cast(values[4] >> 20); + *ptr++ = static_cast(values[4] >> 12); + *ptr++ = static_cast(values[4] >> 4); + *ptr = static_cast(values[4] << 4); + *ptr++ |= static_cast(values[5] >> 24); + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + *ptr++ = static_cast(values[6] >> 20); + *ptr++ = static_cast(values[6] >> 12); + *ptr++ = static_cast(values[6] >> 4); + *ptr = static_cast(values[6] << 4); + *ptr++ |= static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits29(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 21); + + *ptr++ = static_cast(values[0] >> 13); + + *ptr++ = static_cast(values[0] >> 5); + + *ptr = static_cast(values[0] << 3); + *ptr++ |= static_cast(values[1] >> 26); + + *ptr++ = static_cast(values[1] >> 18); + + *ptr++ = static_cast(values[1] >> 10); + + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 23); + + *ptr++ = static_cast(values[2] >> 15); + + *ptr++ = static_cast(values[2] >> 7); + + *ptr = static_cast(values[2] << 1); + *ptr++ |= static_cast(values[3] >> 28); + + *ptr++ = static_cast(values[3] >> 20); + + *ptr++ = static_cast(values[3] >> 12); + + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 25); + + *ptr++ = static_cast(values[4] >> 17); + + *ptr++ = static_cast(values[4] >> 9); + + *ptr++ = static_cast(values[4] >> 1); + + *ptr = static_cast(values[4] << 7); + *ptr++ |= static_cast(values[5] >> 22); + + *ptr++ = static_cast(values[5] >> 14); + + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 27); + + *ptr++ = static_cast(values[6] >> 19); + + *ptr++ = static_cast(values[6] >> 11); + + *ptr++ = static_cast(values[6] >> 3); + + *ptr = static_cast(values[6] << 5); + *ptr++ |= static_cast(values[7] >> 24); + + *ptr++ = static_cast(values[7] >> 16); + + *ptr++ = static_cast(values[7] >> 8); + + *ptr = static_cast(values[7]); +} + +static inline void packBits30(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 22); + *ptr++ = static_cast(values[0] >> 14); + *ptr++ = static_cast(values[0] >> 6); + + *ptr = static_cast(values[0] << 2); + *ptr++ |= static_cast(values[1] >> 28); + *ptr++ = static_cast(values[1] >> 20); + *ptr++ = static_cast(values[1] >> 12); + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 26); + *ptr++ = static_cast(values[2] >> 18); + *ptr++ = static_cast(values[2] >> 10); + *ptr++ = static_cast(values[2] >> 2); + + *ptr = static_cast(values[2] << 6); + *ptr++ |= static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 22); + *ptr++ = static_cast(values[4] >> 14); + *ptr++ = static_cast(values[4] >> 6); + + *ptr = static_cast(values[4] << 2); + *ptr++ |= static_cast(values[5] >> 28); + *ptr++ = static_cast(values[5] >> 20); + *ptr++ = static_cast(values[5] >> 12); + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 26); + *ptr++ = static_cast(values[6] >> 18); + *ptr++ = static_cast(values[6] >> 10); + *ptr++ = static_cast(values[6] >> 2); + + *ptr = static_cast(values[6] << 6); + *ptr++ |= static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits31(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 23); + *ptr++ = static_cast(values[0] >> 15); + *ptr++ = static_cast(values[0] >> 7); + + *ptr = static_cast(values[0] << 1); + *ptr++ |= static_cast(values[1] >> 30); + *ptr++ = static_cast(values[1] >> 22); + *ptr++ = static_cast(values[1] >> 14); + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 29); + *ptr++ = static_cast(values[2] >> 21); + *ptr++ = static_cast(values[2] >> 13); + *ptr++ = static_cast(values[2] >> 5); + + *ptr = static_cast(values[2] << 3); + *ptr++ |= static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 27); + *ptr++ = static_cast(values[4] >> 19); + *ptr++ = static_cast(values[4] >> 11); + *ptr++ = static_cast(values[4] >> 3); + + *ptr = static_cast(values[4] << 5); + *ptr++ |= static_cast(values[5] >> 26); + *ptr++ = static_cast(values[5] >> 18); + *ptr++ = static_cast(values[5] >> 10); + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 25); + *ptr++ = static_cast(values[6] >> 17); + *ptr++ = static_cast(values[6] >> 9); + *ptr++ = static_cast(values[6] >> 1); + + *ptr = static_cast(values[6] << 7); + *ptr++ |= static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits32(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 24); + *ptr++ = static_cast(values[0] >> 16); + *ptr++ = static_cast(values[0] >> 8); + *ptr++ = static_cast(values[0]); + + *ptr++ = static_cast(values[1] >> 24); + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 24); + *ptr++ = static_cast(values[2] >> 16); + *ptr++ = static_cast(values[2] >> 8); + *ptr++ = static_cast(values[2]); + + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 24); + *ptr++ = static_cast(values[4] >> 16); + *ptr++ = static_cast(values[4] >> 8); + *ptr++ = static_cast(values[4]); + + *ptr++ = static_cast(values[5] >> 24); + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 24); + *ptr++ = static_cast(values[6] >> 16); + *ptr++ = static_cast(values[6] >> 8); + *ptr++ = static_cast(values[6]); + + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits33(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 25); + *ptr++ = static_cast(values[0] >> 17); + *ptr++ = static_cast(values[0] >> 9); + *ptr++ = static_cast(values[0] >> 1); + + *ptr = static_cast(values[0] << 7); + *ptr++ |= static_cast(values[1] >> 26); + *ptr++ = static_cast(values[1] >> 18); + *ptr++ = static_cast(values[1] >> 10); + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 27); + *ptr++ = static_cast(values[2] >> 19); + *ptr++ = static_cast(values[2] >> 11); + *ptr++ = static_cast(values[2] >> 3); + + *ptr = static_cast(values[2] << 5); + *ptr++ |= static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 29); + *ptr++ = static_cast(values[4] >> 21); + *ptr++ = static_cast(values[4] >> 13); + *ptr++ = static_cast(values[4] >> 5); + + *ptr = static_cast(values[4] << 3); + *ptr++ |= static_cast(values[5] >> 30); + *ptr++ = static_cast(values[5] >> 22); + *ptr++ = static_cast(values[5] >> 14); + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 31); + *ptr++ = static_cast(values[6] >> 23); + *ptr++ = static_cast(values[6] >> 15); + *ptr++ = static_cast(values[6] >> 7); + + *ptr = static_cast(values[6] << 1); + *ptr++ |= static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits34(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 26); + *ptr++ = static_cast(values[0] >> 18); + *ptr++ = static_cast(values[0] >> 10); + *ptr++ = static_cast(values[0] >> 2); + + *ptr = static_cast(values[0] << 6); + *ptr++ |= static_cast(values[1] >> 28); + *ptr++ = static_cast(values[1] >> 20); + *ptr++ = static_cast(values[1] >> 12); + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 30); + *ptr++ = static_cast(values[2] >> 22); + *ptr++ = static_cast(values[2] >> 14); + *ptr++ = static_cast(values[2] >> 6); + + *ptr = static_cast(values[2] << 2); + *ptr++ |= static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 26); + *ptr++ = static_cast(values[4] >> 18); + *ptr++ = static_cast(values[4] >> 10); + *ptr++ = static_cast(values[4] >> 2); + + *ptr = static_cast(values[4] << 6); + *ptr++ |= static_cast(values[5] >> 28); + *ptr++ = static_cast(values[5] >> 20); + *ptr++ = static_cast(values[5] >> 12); + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 30); + *ptr++ = static_cast(values[6] >> 22); + *ptr++ = static_cast(values[6] >> 14); + *ptr++ = static_cast(values[6] >> 6); + + *ptr = static_cast(values[6] << 2); + *ptr++ |= static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits35(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 27); + *ptr++ = static_cast(values[0] >> 19); + *ptr++ = static_cast(values[0] >> 11); + *ptr++ = static_cast(values[0] >> 3); + + *ptr = static_cast(values[0] << 5); + *ptr++ |= static_cast(values[1] >> 30); + *ptr++ = static_cast(values[1] >> 22); + *ptr++ = static_cast(values[1] >> 14); + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 33); + *ptr++ = static_cast(values[2] >> 25); + *ptr++ = static_cast(values[2] >> 17); + *ptr++ = static_cast(values[2] >> 9); + *ptr++ = static_cast(values[2] >> 1); + + *ptr = static_cast(values[2] << 7); + *ptr++ |= static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 31); + *ptr++ = static_cast(values[4] >> 23); + *ptr++ = static_cast(values[4] >> 15); + *ptr++ = static_cast(values[4] >> 7); + + *ptr = static_cast(values[4] << 1); + *ptr++ |= static_cast(values[5] >> 34); + *ptr++ = static_cast(values[5] >> 26); + *ptr++ = static_cast(values[5] >> 18); + *ptr++ = static_cast(values[5] >> 10); + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 29); + *ptr++ = static_cast(values[6] >> 21); + *ptr++ = static_cast(values[6] >> 13); + *ptr++ = static_cast(values[6] >> 5); + + *ptr = static_cast(values[6] << 3); + *ptr++ |= static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits36(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 28); + *ptr++ = static_cast(values[0] >> 20); + *ptr++ = static_cast(values[0] >> 12); + *ptr++ = static_cast(values[0] >> 4); + + *ptr = static_cast(values[0] << 4); + *ptr++ |= static_cast(values[1] >> 32); + *ptr++ = static_cast(values[1] >> 24); + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 28); + *ptr++ = static_cast(values[2] >> 20); + *ptr++ = static_cast(values[2] >> 12); + *ptr++ = static_cast(values[2] >> 4); + + *ptr = static_cast(values[2] << 4); + *ptr++ |= static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 28); + *ptr++ = static_cast(values[4] >> 20); + *ptr++ = static_cast(values[4] >> 12); + *ptr++ = static_cast(values[4] >> 4); + + *ptr = static_cast(values[4] << 4); + *ptr++ |= static_cast(values[5] >> 32); + *ptr++ = static_cast(values[5] >> 24); + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 28); + *ptr++ = static_cast(values[6] >> 20); + *ptr++ = static_cast(values[6] >> 12); + *ptr++ = static_cast(values[6] >> 4); + + *ptr = static_cast(values[6] << 4); + *ptr++ |= static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits37(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 29); + *ptr++ = static_cast(values[0] >> 21); + *ptr++ = static_cast(values[0] >> 13); + *ptr++ = static_cast(values[0] >> 5); + + *ptr = static_cast(values[0] << 3); + *ptr++ |= static_cast(values[1] >> 34); + *ptr++ = static_cast(values[1] >> 26); + *ptr++ = static_cast(values[1] >> 18); + *ptr++ = static_cast(values[1] >> 10); + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 31); + *ptr++ = static_cast(values[2] >> 23); + *ptr++ = static_cast(values[2] >> 15); + *ptr++ = static_cast(values[2] >> 7); + + *ptr = static_cast(values[2] << 1); + *ptr++ |= static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 33); + *ptr++ = static_cast(values[4] >> 25); + *ptr++ = static_cast(values[4] >> 17); + *ptr++ = static_cast(values[4] >> 9); + *ptr++ = static_cast(values[4] >> 1); + + *ptr = static_cast(values[4] << 7); + *ptr++ |= static_cast(values[5] >> 30); + *ptr++ = static_cast(values[5] >> 22); + *ptr++ = static_cast(values[5] >> 14); + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 35); + *ptr++ = static_cast(values[6] >> 27); + *ptr++ = static_cast(values[6] >> 19); + *ptr++ = static_cast(values[6] >> 11); + *ptr++ = static_cast(values[6] >> 3); + + *ptr = static_cast(values[6] << 5); + *ptr++ |= static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits38(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 30); + *ptr++ = static_cast(values[0] >> 22); + *ptr++ = static_cast(values[0] >> 14); + *ptr++ = static_cast(values[0] >> 6); + + *ptr = static_cast(values[0] << 2); + *ptr++ |= static_cast(values[1] >> 36); + *ptr++ = static_cast(values[1] >> 28); + *ptr++ = static_cast(values[1] >> 20); + *ptr++ = static_cast(values[1] >> 12); + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 34); + *ptr++ = static_cast(values[2] >> 26); + *ptr++ = static_cast(values[2] >> 18); + *ptr++ = static_cast(values[2] >> 10); + *ptr++ = static_cast(values[2] >> 2); + + *ptr = static_cast(values[2] << 6); + *ptr++ |= static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 30); + *ptr++ = static_cast(values[4] >> 22); + *ptr++ = static_cast(values[4] >> 14); + *ptr++ = static_cast(values[4] >> 6); + + *ptr = static_cast(values[4] << 2); + *ptr++ |= static_cast(values[5] >> 36); + *ptr++ = static_cast(values[5] >> 28); + *ptr++ = static_cast(values[5] >> 20); + *ptr++ = static_cast(values[5] >> 12); + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 34); + *ptr++ = static_cast(values[6] >> 26); + *ptr++ = static_cast(values[6] >> 18); + *ptr++ = static_cast(values[6] >> 10); + *ptr++ = static_cast(values[6] >> 2); + + *ptr = static_cast(values[6] << 6); + *ptr++ |= static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits39(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 31); + *ptr++ = static_cast(values[0] >> 23); + *ptr++ = static_cast(values[0] >> 15); + *ptr++ = static_cast(values[0] >> 7); + + *ptr = static_cast(values[0] << 1); + *ptr++ |= static_cast(values[1] >> 38); + *ptr++ = static_cast(values[1] >> 30); + *ptr++ = static_cast(values[1] >> 22); + *ptr++ = static_cast(values[1] >> 14); + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 37); + *ptr++ = static_cast(values[2] >> 29); + *ptr++ = static_cast(values[2] >> 21); + *ptr++ = static_cast(values[2] >> 13); + *ptr++ = static_cast(values[2] >> 5); + + *ptr = static_cast(values[2] << 3); + *ptr++ |= static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 35); + *ptr++ = static_cast(values[4] >> 27); + *ptr++ = static_cast(values[4] >> 19); + *ptr++ = static_cast(values[4] >> 11); + *ptr++ = static_cast(values[4] >> 3); + + *ptr = static_cast(values[4] << 5); + *ptr++ |= static_cast(values[5] >> 34); + *ptr++ = static_cast(values[5] >> 26); + *ptr++ = static_cast(values[5] >> 18); + *ptr++ = static_cast(values[5] >> 10); + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 33); + *ptr++ = static_cast(values[6] >> 25); + *ptr++ = static_cast(values[6] >> 17); + *ptr++ = static_cast(values[6] >> 9); + *ptr++ = static_cast(values[6] >> 1); + + *ptr = static_cast(values[6] << 7); + *ptr++ |= static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits40(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 32); + *ptr++ = static_cast(values[0] >> 24); + *ptr++ = static_cast(values[0] >> 16); + *ptr++ = static_cast(values[0] >> 8); + *ptr++ = static_cast(values[0]); + + *ptr++ = static_cast(values[1] >> 32); + *ptr++ = static_cast(values[1] >> 24); + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 32); + *ptr++ = static_cast(values[2] >> 24); + *ptr++ = static_cast(values[2] >> 16); + *ptr++ = static_cast(values[2] >> 8); + *ptr++ = static_cast(values[2]); + + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 32); + *ptr++ = static_cast(values[4] >> 24); + *ptr++ = static_cast(values[4] >> 16); + *ptr++ = static_cast(values[4] >> 8); + *ptr++ = static_cast(values[4]); + + *ptr++ = static_cast(values[5] >> 32); + *ptr++ = static_cast(values[5] >> 24); + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 32); + *ptr++ = static_cast(values[6] >> 24); + *ptr++ = static_cast(values[6] >> 16); + *ptr++ = static_cast(values[6] >> 8); + *ptr++ = static_cast(values[6]); + + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits41(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 33); + *ptr++ = static_cast(values[0] >> 25); + *ptr++ = static_cast(values[0] >> 17); + *ptr++ = static_cast(values[0] >> 9); + *ptr++ = static_cast(values[0] >> 1); + + *ptr = static_cast(values[0] << 7); + *ptr++ |= static_cast(values[1] >> 34); + *ptr++ = static_cast(values[1] >> 26); + *ptr++ = static_cast(values[1] >> 18); + *ptr++ = static_cast(values[1] >> 10); + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 35); + *ptr++ = static_cast(values[2] >> 27); + *ptr++ = static_cast(values[2] >> 19); + *ptr++ = static_cast(values[2] >> 11); + *ptr++ = static_cast(values[2] >> 3); + + *ptr = static_cast(values[2] << 5); + *ptr++ |= static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 37); + *ptr++ = static_cast(values[4] >> 29); + *ptr++ = static_cast(values[4] >> 21); + *ptr++ = static_cast(values[4] >> 13); + *ptr++ = static_cast(values[4] >> 5); + + *ptr = static_cast(values[4] << 3); + *ptr++ |= static_cast(values[5] >> 38); + *ptr++ = static_cast(values[5] >> 30); + *ptr++ = static_cast(values[5] >> 22); + *ptr++ = static_cast(values[5] >> 14); + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 39); + *ptr++ = static_cast(values[6] >> 31); + *ptr++ = static_cast(values[6] >> 23); + *ptr++ = static_cast(values[6] >> 15); + *ptr++ = static_cast(values[6] >> 7); + + *ptr = static_cast(values[6] << 1); + *ptr++ |= static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits42(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 34); + *ptr++ = static_cast(values[0] >> 26); + *ptr++ = static_cast(values[0] >> 18); + *ptr++ = static_cast(values[0] >> 10); + *ptr++ = static_cast(values[0] >> 2); + + *ptr = static_cast(values[0] << 6); + *ptr++ |= static_cast(values[1] >> 36); + *ptr++ = static_cast(values[1] >> 28); + *ptr++ = static_cast(values[1] >> 20); + *ptr++ = static_cast(values[1] >> 12); + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 38); + *ptr++ = static_cast(values[2] >> 30); + *ptr++ = static_cast(values[2] >> 22); + *ptr++ = static_cast(values[2] >> 14); + *ptr++ = static_cast(values[2] >> 6); + + *ptr = static_cast(values[2] << 2); + *ptr++ |= static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 34); + *ptr++ = static_cast(values[4] >> 26); + *ptr++ = static_cast(values[4] >> 18); + *ptr++ = static_cast(values[4] >> 10); + *ptr++ = static_cast(values[4] >> 2); + + *ptr = static_cast(values[4] << 6); + *ptr++ |= static_cast(values[5] >> 36); + *ptr++ = static_cast(values[5] >> 28); + *ptr++ = static_cast(values[5] >> 20); + *ptr++ = static_cast(values[5] >> 12); + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 38); + *ptr++ = static_cast(values[6] >> 30); + *ptr++ = static_cast(values[6] >> 22); + *ptr++ = static_cast(values[6] >> 14); + *ptr++ = static_cast(values[6] >> 6); + + *ptr = static_cast(values[6] << 2); + *ptr++ |= static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits43(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 35); + *ptr++ = static_cast(values[0] >> 27); + *ptr++ = static_cast(values[0] >> 19); + *ptr++ = static_cast(values[0] >> 11); + *ptr++ = static_cast(values[0] >> 3); + + *ptr = static_cast(values[0] << 5); + *ptr++ |= static_cast(values[1] >> 38); + *ptr++ = static_cast(values[1] >> 30); + *ptr++ = static_cast(values[1] >> 22); + *ptr++ = static_cast(values[1] >> 14); + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 41); + *ptr++ = static_cast(values[2] >> 33); + *ptr++ = static_cast(values[2] >> 25); + *ptr++ = static_cast(values[2] >> 17); + *ptr++ = static_cast(values[2] >> 9); + *ptr++ = static_cast(values[2] >> 1); + + *ptr = static_cast(values[2] << 7); + *ptr++ |= static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 39); + *ptr++ = static_cast(values[4] >> 31); + *ptr++ = static_cast(values[4] >> 23); + *ptr++ = static_cast(values[4] >> 15); + *ptr++ = static_cast(values[4] >> 7); + + *ptr = static_cast(values[4] << 1); + *ptr++ |= static_cast(values[5] >> 42); + *ptr++ = static_cast(values[5] >> 34); + *ptr++ = static_cast(values[5] >> 26); + *ptr++ = static_cast(values[5] >> 18); + *ptr++ = static_cast(values[5] >> 10); + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 37); + *ptr++ = static_cast(values[6] >> 29); + *ptr++ = static_cast(values[6] >> 21); + *ptr++ = static_cast(values[6] >> 13); + *ptr++ = static_cast(values[6] >> 5); + + *ptr = static_cast(values[6] << 3); + *ptr++ |= static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits44(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 36); + *ptr++ = static_cast(values[0] >> 28); + *ptr++ = static_cast(values[0] >> 20); + *ptr++ = static_cast(values[0] >> 12); + *ptr++ = static_cast(values[0] >> 4); + + *ptr = static_cast(values[0] << 4); + *ptr++ |= static_cast(values[1] >> 40); + *ptr++ = static_cast(values[1] >> 32); + *ptr++ = static_cast(values[1] >> 24); + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 36); + *ptr++ = static_cast(values[2] >> 28); + *ptr++ = static_cast(values[2] >> 20); + *ptr++ = static_cast(values[2] >> 12); + *ptr++ = static_cast(values[2] >> 4); + + *ptr = static_cast(values[2] << 4); + *ptr++ |= static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 36); + *ptr++ = static_cast(values[4] >> 28); + *ptr++ = static_cast(values[4] >> 20); + *ptr++ = static_cast(values[4] >> 12); + *ptr++ = static_cast(values[4] >> 4); + + *ptr = static_cast(values[4] << 4); + *ptr++ |= static_cast(values[5] >> 40); + *ptr++ = static_cast(values[5] >> 32); + *ptr++ = static_cast(values[5] >> 24); + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 36); + *ptr++ = static_cast(values[6] >> 28); + *ptr++ = static_cast(values[6] >> 20); + *ptr++ = static_cast(values[6] >> 12); + *ptr++ = static_cast(values[6] >> 4); + + *ptr = static_cast(values[6] << 4); + *ptr++ |= static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits45(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 37); + *ptr++ = static_cast(values[0] >> 29); + *ptr++ = static_cast(values[0] >> 21); + *ptr++ = static_cast(values[0] >> 13); + *ptr++ = static_cast(values[0] >> 5); + + *ptr = static_cast(values[0] << 3); + *ptr++ |= static_cast(values[1] >> 42); + *ptr++ = static_cast(values[1] >> 34); + *ptr++ = static_cast(values[1] >> 26); + *ptr++ = static_cast(values[1] >> 18); + *ptr++ = static_cast(values[1] >> 10); + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 39); + *ptr++ = static_cast(values[2] >> 31); + *ptr++ = static_cast(values[2] >> 23); + *ptr++ = static_cast(values[2] >> 15); + *ptr++ = static_cast(values[2] >> 7); + + *ptr = static_cast(values[2] << 1); + *ptr++ |= static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 41); + *ptr++ = static_cast(values[4] >> 33); + *ptr++ = static_cast(values[4] >> 25); + *ptr++ = static_cast(values[4] >> 17); + *ptr++ = static_cast(values[4] >> 9); + *ptr++ = static_cast(values[4] >> 1); + + *ptr = static_cast(values[4] << 7); + *ptr++ |= static_cast(values[5] >> 38); + *ptr++ = static_cast(values[5] >> 30); + *ptr++ = static_cast(values[5] >> 22); + *ptr++ = static_cast(values[5] >> 14); + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 43); + *ptr++ = static_cast(values[6] >> 35); + *ptr++ = static_cast(values[6] >> 27); + *ptr++ = static_cast(values[6] >> 19); + *ptr++ = static_cast(values[6] >> 11); + *ptr++ = static_cast(values[6] >> 3); + + *ptr = static_cast(values[6] << 5); + *ptr++ |= static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits46(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 38); + *ptr++ = static_cast(values[0] >> 30); + *ptr++ = static_cast(values[0] >> 22); + *ptr++ = static_cast(values[0] >> 14); + *ptr++ = static_cast(values[0] >> 6); + + *ptr = static_cast(values[0] << 2); + *ptr++ |= static_cast(values[1] >> 44); + *ptr++ = static_cast(values[1] >> 36); + *ptr++ = static_cast(values[1] >> 28); + *ptr++ = static_cast(values[1] >> 20); + *ptr++ = static_cast(values[1] >> 12); + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 42); + *ptr++ = static_cast(values[2] >> 34); + *ptr++ = static_cast(values[2] >> 26); + *ptr++ = static_cast(values[2] >> 18); + *ptr++ = static_cast(values[2] >> 10); + *ptr++ = static_cast(values[2] >> 2); + + *ptr = static_cast(values[2] << 6); + *ptr++ |= static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 38); + *ptr++ = static_cast(values[4] >> 30); + *ptr++ = static_cast(values[4] >> 22); + *ptr++ = static_cast(values[4] >> 14); + *ptr++ = static_cast(values[4] >> 6); + + *ptr = static_cast(values[4] << 2); + *ptr++ |= static_cast(values[5] >> 44); + *ptr++ = static_cast(values[5] >> 36); + *ptr++ = static_cast(values[5] >> 28); + *ptr++ = static_cast(values[5] >> 20); + *ptr++ = static_cast(values[5] >> 12); + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 42); + *ptr++ = static_cast(values[6] >> 34); + *ptr++ = static_cast(values[6] >> 26); + *ptr++ = static_cast(values[6] >> 18); + *ptr++ = static_cast(values[6] >> 10); + *ptr++ = static_cast(values[6] >> 2); + + *ptr = static_cast(values[6] << 6); + *ptr++ |= static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits47(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 39); + *ptr++ = static_cast(values[0] >> 31); + *ptr++ = static_cast(values[0] >> 23); + *ptr++ = static_cast(values[0] >> 15); + *ptr++ = static_cast(values[0] >> 7); + + *ptr = static_cast(values[0] << 1); + *ptr++ |= static_cast(values[1] >> 46); + *ptr++ = static_cast(values[1] >> 38); + *ptr++ = static_cast(values[1] >> 30); + *ptr++ = static_cast(values[1] >> 22); + *ptr++ = static_cast(values[1] >> 14); + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 45); + *ptr++ = static_cast(values[2] >> 37); + *ptr++ = static_cast(values[2] >> 29); + *ptr++ = static_cast(values[2] >> 21); + *ptr++ = static_cast(values[2] >> 13); + *ptr++ = static_cast(values[2] >> 5); + + *ptr = static_cast(values[2] << 3); + *ptr++ |= static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 43); + *ptr++ = static_cast(values[4] >> 35); + *ptr++ = static_cast(values[4] >> 27); + *ptr++ = static_cast(values[4] >> 19); + *ptr++ = static_cast(values[4] >> 11); + *ptr++ = static_cast(values[4] >> 3); + + *ptr = static_cast(values[4] << 5); + *ptr++ |= static_cast(values[5] >> 42); + *ptr++ = static_cast(values[5] >> 34); + *ptr++ = static_cast(values[5] >> 26); + *ptr++ = static_cast(values[5] >> 18); + *ptr++ = static_cast(values[5] >> 10); + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 41); + *ptr++ = static_cast(values[6] >> 33); + *ptr++ = static_cast(values[6] >> 25); + *ptr++ = static_cast(values[6] >> 17); + *ptr++ = static_cast(values[6] >> 9); + *ptr++ = static_cast(values[6] >> 1); + + *ptr = static_cast(values[6] << 7); + *ptr++ |= static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits48(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 40); + *ptr++ = static_cast(values[0] >> 32); + *ptr++ = static_cast(values[0] >> 24); + *ptr++ = static_cast(values[0] >> 16); + *ptr++ = static_cast(values[0] >> 8); + *ptr++ = static_cast(values[0]); + + *ptr++ = static_cast(values[1] >> 40); + *ptr++ = static_cast(values[1] >> 32); + *ptr++ = static_cast(values[1] >> 24); + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 40); + *ptr++ = static_cast(values[2] >> 32); + *ptr++ = static_cast(values[2] >> 24); + *ptr++ = static_cast(values[2] >> 16); + *ptr++ = static_cast(values[2] >> 8); + *ptr++ = static_cast(values[2]); + + *ptr++ = static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 40); + *ptr++ = static_cast(values[4] >> 32); + *ptr++ = static_cast(values[4] >> 24); + *ptr++ = static_cast(values[4] >> 16); + *ptr++ = static_cast(values[4] >> 8); + *ptr++ = static_cast(values[4]); + + *ptr++ = static_cast(values[5] >> 40); + *ptr++ = static_cast(values[5] >> 32); + *ptr++ = static_cast(values[5] >> 24); + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 40); + *ptr++ = static_cast(values[6] >> 32); + *ptr++ = static_cast(values[6] >> 24); + *ptr++ = static_cast(values[6] >> 16); + *ptr++ = static_cast(values[6] >> 8); + *ptr++ = static_cast(values[6]); + + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits49(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 41); + *ptr++ = static_cast(values[0] >> 33); + *ptr++ = static_cast(values[0] >> 25); + *ptr++ = static_cast(values[0] >> 17); + *ptr++ = static_cast(values[0] >> 9); + *ptr++ = static_cast(values[0] >> 1); + + *ptr = static_cast(values[0] << 7); + *ptr++ |= static_cast(values[1] >> 42); + *ptr++ = static_cast(values[1] >> 34); + *ptr++ = static_cast(values[1] >> 26); + *ptr++ = static_cast(values[1] >> 18); + *ptr++ = static_cast(values[1] >> 10); + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 43); + *ptr++ = static_cast(values[2] >> 35); + *ptr++ = static_cast(values[2] >> 27); + *ptr++ = static_cast(values[2] >> 19); + *ptr++ = static_cast(values[2] >> 11); + *ptr++ = static_cast(values[2] >> 3); + + *ptr = static_cast(values[2] << 5); + *ptr++ |= static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 45); + *ptr++ = static_cast(values[4] >> 37); + *ptr++ = static_cast(values[4] >> 29); + *ptr++ = static_cast(values[4] >> 21); + *ptr++ = static_cast(values[4] >> 13); + *ptr++ = static_cast(values[4] >> 5); + + *ptr = static_cast(values[4] << 3); + *ptr++ |= static_cast(values[5] >> 46); + *ptr++ = static_cast(values[5] >> 38); + *ptr++ = static_cast(values[5] >> 30); + *ptr++ = static_cast(values[5] >> 22); + *ptr++ = static_cast(values[5] >> 14); + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 47); + *ptr++ = static_cast(values[6] >> 39); + *ptr++ = static_cast(values[6] >> 31); + *ptr++ = static_cast(values[6] >> 23); + *ptr++ = static_cast(values[6] >> 15); + *ptr++ = static_cast(values[6] >> 7); + + *ptr = static_cast(values[6] << 1); + *ptr++ |= static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits50(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 42); + *ptr++ = static_cast(values[0] >> 34); + *ptr++ = static_cast(values[0] >> 26); + *ptr++ = static_cast(values[0] >> 18); + *ptr++ = static_cast(values[0] >> 10); + *ptr++ = static_cast(values[0] >> 2); + + *ptr = static_cast(values[0] << 6); + *ptr++ |= static_cast(values[1] >> 44); + *ptr++ = static_cast(values[1] >> 36); + *ptr++ = static_cast(values[1] >> 28); + *ptr++ = static_cast(values[1] >> 20); + *ptr++ = static_cast(values[1] >> 12); + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 46); + *ptr++ = static_cast(values[2] >> 38); + *ptr++ = static_cast(values[2] >> 30); + *ptr++ = static_cast(values[2] >> 22); + *ptr++ = static_cast(values[2] >> 14); + *ptr++ = static_cast(values[2] >> 6); + + *ptr = static_cast(values[2] << 2); + *ptr++ |= static_cast(values[3] >> 48); + *ptr++ = static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 42); + *ptr++ = static_cast(values[4] >> 34); + *ptr++ = static_cast(values[4] >> 26); + *ptr++ = static_cast(values[4] >> 18); + *ptr++ = static_cast(values[4] >> 10); + *ptr++ = static_cast(values[4] >> 2); + + *ptr = static_cast(values[4] << 6); + *ptr++ |= static_cast(values[5] >> 44); + *ptr++ = static_cast(values[5] >> 36); + *ptr++ = static_cast(values[5] >> 28); + *ptr++ = static_cast(values[5] >> 20); + *ptr++ = static_cast(values[5] >> 12); + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 46); + *ptr++ = static_cast(values[6] >> 38); + *ptr++ = static_cast(values[6] >> 30); + *ptr++ = static_cast(values[6] >> 22); + *ptr++ = static_cast(values[6] >> 14); + *ptr++ = static_cast(values[6] >> 6); + + *ptr = static_cast(values[6] << 2); + *ptr++ |= static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits51(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 43); + *ptr++ = static_cast(values[0] >> 35); + *ptr++ = static_cast(values[0] >> 27); + *ptr++ = static_cast(values[0] >> 19); + *ptr++ = static_cast(values[0] >> 11); + *ptr++ = static_cast(values[0] >> 3); + + *ptr = static_cast(values[0] << 5); + *ptr++ |= static_cast(values[1] >> 46); + *ptr++ = static_cast(values[1] >> 38); + *ptr++ = static_cast(values[1] >> 30); + *ptr++ = static_cast(values[1] >> 22); + *ptr++ = static_cast(values[1] >> 14); + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 49); + *ptr++ = static_cast(values[2] >> 41); + *ptr++ = static_cast(values[2] >> 33); + *ptr++ = static_cast(values[2] >> 25); + *ptr++ = static_cast(values[2] >> 17); + *ptr++ = static_cast(values[2] >> 9); + *ptr++ = static_cast(values[2] >> 1); + + *ptr = static_cast(values[2] << 7); + *ptr++ |= static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 47); + *ptr++ = static_cast(values[4] >> 39); + *ptr++ = static_cast(values[4] >> 31); + *ptr++ = static_cast(values[4] >> 23); + *ptr++ = static_cast(values[4] >> 15); + *ptr++ = static_cast(values[4] >> 7); + + *ptr = static_cast(values[4] << 1); + *ptr++ |= static_cast(values[5] >> 50); + *ptr++ = static_cast(values[5] >> 42); + *ptr++ = static_cast(values[5] >> 34); + *ptr++ = static_cast(values[5] >> 26); + *ptr++ = static_cast(values[5] >> 18); + *ptr++ = static_cast(values[5] >> 10); + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 45); + *ptr++ = static_cast(values[6] >> 37); + *ptr++ = static_cast(values[6] >> 29); + *ptr++ = static_cast(values[6] >> 21); + *ptr++ = static_cast(values[6] >> 13); + *ptr++ = static_cast(values[6] >> 5); + + *ptr = static_cast(values[6] << 3); + *ptr++ |= static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits52(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 44); + *ptr++ = static_cast(values[0] >> 36); + *ptr++ = static_cast(values[0] >> 28); + *ptr++ = static_cast(values[0] >> 20); + *ptr++ = static_cast(values[0] >> 12); + *ptr++ = static_cast(values[0] >> 4); + + *ptr = static_cast(values[0] << 4); + *ptr++ |= static_cast(values[1] >> 48); + *ptr++ = static_cast(values[1] >> 40); + *ptr++ = static_cast(values[1] >> 32); + *ptr++ = static_cast(values[1] >> 24); + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 44); + *ptr++ = static_cast(values[2] >> 36); + *ptr++ = static_cast(values[2] >> 28); + *ptr++ = static_cast(values[2] >> 20); + *ptr++ = static_cast(values[2] >> 12); + *ptr++ = static_cast(values[2] >> 4); + + *ptr = static_cast(values[2] << 4); + *ptr++ |= static_cast(values[3] >> 48); + *ptr++ = static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 44); + *ptr++ = static_cast(values[4] >> 36); + *ptr++ = static_cast(values[4] >> 28); + *ptr++ = static_cast(values[4] >> 20); + *ptr++ = static_cast(values[4] >> 12); + *ptr++ = static_cast(values[4] >> 4); + + *ptr = static_cast(values[4] << 4); + *ptr++ |= static_cast(values[5] >> 48); + *ptr++ = static_cast(values[5] >> 40); + *ptr++ = static_cast(values[5] >> 32); + *ptr++ = static_cast(values[5] >> 24); + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 44); + *ptr++ = static_cast(values[6] >> 36); + *ptr++ = static_cast(values[6] >> 28); + *ptr++ = static_cast(values[6] >> 20); + *ptr++ = static_cast(values[6] >> 12); + *ptr++ = static_cast(values[6] >> 4); + + *ptr = static_cast(values[6] << 4); + *ptr++ |= static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits53(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 45); + *ptr++ = static_cast(values[0] >> 37); + *ptr++ = static_cast(values[0] >> 29); + *ptr++ = static_cast(values[0] >> 21); + *ptr++ = static_cast(values[0] >> 13); + *ptr++ = static_cast(values[0] >> 5); + + *ptr = static_cast(values[0] << 3); + *ptr++ |= static_cast(values[1] >> 50); + *ptr++ = static_cast(values[1] >> 42); + *ptr++ = static_cast(values[1] >> 34); + *ptr++ = static_cast(values[1] >> 26); + *ptr++ = static_cast(values[1] >> 18); + *ptr++ = static_cast(values[1] >> 10); + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 47); + *ptr++ = static_cast(values[2] >> 39); + *ptr++ = static_cast(values[2] >> 31); + *ptr++ = static_cast(values[2] >> 23); + *ptr++ = static_cast(values[2] >> 15); + *ptr++ = static_cast(values[2] >> 7); + + *ptr = static_cast(values[2] << 1); + *ptr++ |= static_cast(values[3] >> 52); + *ptr++ = static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 49); + *ptr++ = static_cast(values[4] >> 41); + *ptr++ = static_cast(values[4] >> 33); + *ptr++ = static_cast(values[4] >> 25); + *ptr++ = static_cast(values[4] >> 17); + *ptr++ = static_cast(values[4] >> 9); + *ptr++ = static_cast(values[4] >> 1); + + *ptr = static_cast(values[4] << 7); + *ptr++ |= static_cast(values[5] >> 46); + *ptr++ = static_cast(values[5] >> 38); + *ptr++ = static_cast(values[5] >> 30); + *ptr++ = static_cast(values[5] >> 22); + *ptr++ = static_cast(values[5] >> 14); + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 51); + *ptr++ = static_cast(values[6] >> 43); + *ptr++ = static_cast(values[6] >> 35); + *ptr++ = static_cast(values[6] >> 27); + *ptr++ = static_cast(values[6] >> 19); + *ptr++ = static_cast(values[6] >> 11); + *ptr++ = static_cast(values[6] >> 3); + + *ptr = static_cast(values[6] << 5); + *ptr++ |= static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits54(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 46); + *ptr++ = static_cast(values[0] >> 38); + *ptr++ = static_cast(values[0] >> 30); + *ptr++ = static_cast(values[0] >> 22); + *ptr++ = static_cast(values[0] >> 14); + *ptr++ = static_cast(values[0] >> 6); + + *ptr = static_cast(values[0] << 2); + *ptr++ |= static_cast(values[1] >> 52); + *ptr++ = static_cast(values[1] >> 44); + *ptr++ = static_cast(values[1] >> 36); + *ptr++ = static_cast(values[1] >> 28); + *ptr++ = static_cast(values[1] >> 20); + *ptr++ = static_cast(values[1] >> 12); + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 50); + *ptr++ = static_cast(values[2] >> 42); + *ptr++ = static_cast(values[2] >> 34); + *ptr++ = static_cast(values[2] >> 26); + *ptr++ = static_cast(values[2] >> 18); + *ptr++ = static_cast(values[2] >> 10); + *ptr++ = static_cast(values[2] >> 2); + + *ptr = static_cast(values[2] << 6); + *ptr++ |= static_cast(values[3] >> 48); + *ptr++ = static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 46); + *ptr++ = static_cast(values[4] >> 38); + *ptr++ = static_cast(values[4] >> 30); + *ptr++ = static_cast(values[4] >> 22); + *ptr++ = static_cast(values[4] >> 14); + *ptr++ = static_cast(values[4] >> 6); + + *ptr = static_cast(values[4] << 2); + *ptr++ |= static_cast(values[5] >> 52); + *ptr++ = static_cast(values[5] >> 44); + *ptr++ = static_cast(values[5] >> 36); + *ptr++ = static_cast(values[5] >> 28); + *ptr++ = static_cast(values[5] >> 20); + *ptr++ = static_cast(values[5] >> 12); + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 50); + *ptr++ = static_cast(values[6] >> 42); + *ptr++ = static_cast(values[6] >> 34); + *ptr++ = static_cast(values[6] >> 26); + *ptr++ = static_cast(values[6] >> 18); + *ptr++ = static_cast(values[6] >> 10); + *ptr++ = static_cast(values[6] >> 2); + + *ptr = static_cast(values[6] << 6); + *ptr++ |= static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits55(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 47); + *ptr++ = static_cast(values[0] >> 39); + *ptr++ = static_cast(values[0] >> 31); + *ptr++ = static_cast(values[0] >> 23); + *ptr++ = static_cast(values[0] >> 15); + *ptr++ = static_cast(values[0] >> 7); + + *ptr = static_cast(values[0] << 1); + *ptr++ |= static_cast(values[1] >> 54); + *ptr++ = static_cast(values[1] >> 46); + *ptr++ = static_cast(values[1] >> 38); + *ptr++ = static_cast(values[1] >> 30); + *ptr++ = static_cast(values[1] >> 22); + *ptr++ = static_cast(values[1] >> 14); + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 53); + *ptr++ = static_cast(values[2] >> 45); + *ptr++ = static_cast(values[2] >> 37); + *ptr++ = static_cast(values[2] >> 29); + *ptr++ = static_cast(values[2] >> 21); + *ptr++ = static_cast(values[2] >> 13); + *ptr++ = static_cast(values[2] >> 5); + + *ptr = static_cast(values[2] << 3); + *ptr++ |= static_cast(values[3] >> 52); + *ptr++ = static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 51); + *ptr++ = static_cast(values[4] >> 43); + *ptr++ = static_cast(values[4] >> 35); + *ptr++ = static_cast(values[4] >> 27); + *ptr++ = static_cast(values[4] >> 19); + *ptr++ = static_cast(values[4] >> 11); + *ptr++ = static_cast(values[4] >> 3); + + *ptr = static_cast(values[4] << 5); + *ptr++ |= static_cast(values[5] >> 50); + *ptr++ = static_cast(values[5] >> 42); + *ptr++ = static_cast(values[5] >> 34); + *ptr++ = static_cast(values[5] >> 26); + *ptr++ = static_cast(values[5] >> 18); + *ptr++ = static_cast(values[5] >> 10); + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 49); + *ptr++ = static_cast(values[6] >> 41); + *ptr++ = static_cast(values[6] >> 33); + *ptr++ = static_cast(values[6] >> 25); + *ptr++ = static_cast(values[6] >> 17); + *ptr++ = static_cast(values[6] >> 9); + *ptr++ = static_cast(values[6] >> 1); + + *ptr = static_cast(values[6] << 7); + *ptr++ |= static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits56(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 48); + *ptr++ = static_cast(values[0] >> 40); + *ptr++ = static_cast(values[0] >> 32); + *ptr++ = static_cast(values[0] >> 24); + *ptr++ = static_cast(values[0] >> 16); + *ptr++ = static_cast(values[0] >> 8); + *ptr++ = static_cast(values[0]); + + *ptr++ = static_cast(values[1] >> 48); + *ptr++ = static_cast(values[1] >> 40); + *ptr++ = static_cast(values[1] >> 32); + *ptr++ = static_cast(values[1] >> 24); + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 48); + *ptr++ = static_cast(values[2] >> 40); + *ptr++ = static_cast(values[2] >> 32); + *ptr++ = static_cast(values[2] >> 24); + *ptr++ = static_cast(values[2] >> 16); + *ptr++ = static_cast(values[2] >> 8); + *ptr++ = static_cast(values[2]); + + *ptr++ = static_cast(values[3] >> 48); + *ptr++ = static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 48); + *ptr++ = static_cast(values[4] >> 40); + *ptr++ = static_cast(values[4] >> 32); + *ptr++ = static_cast(values[4] >> 24); + *ptr++ = static_cast(values[4] >> 16); + *ptr++ = static_cast(values[4] >> 8); + *ptr++ = static_cast(values[4]); + + *ptr++ = static_cast(values[5] >> 48); + *ptr++ = static_cast(values[5] >> 40); + *ptr++ = static_cast(values[5] >> 32); + *ptr++ = static_cast(values[5] >> 24); + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 48); + *ptr++ = static_cast(values[6] >> 40); + *ptr++ = static_cast(values[6] >> 32); + *ptr++ = static_cast(values[6] >> 24); + *ptr++ = static_cast(values[6] >> 16); + *ptr++ = static_cast(values[6] >> 8); + *ptr++ = static_cast(values[6]); + + *ptr++ = static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits57(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 49); + *ptr++ = static_cast(values[0] >> 41); + *ptr++ = static_cast(values[0] >> 33); + *ptr++ = static_cast(values[0] >> 25); + *ptr++ = static_cast(values[0] >> 17); + *ptr++ = static_cast(values[0] >> 9); + *ptr++ = static_cast(values[0] >> 1); + + *ptr = static_cast(values[0] << 7); + *ptr++ |= static_cast(values[1] >> 50); + *ptr++ = static_cast(values[1] >> 42); + *ptr++ = static_cast(values[1] >> 34); + *ptr++ = static_cast(values[1] >> 26); + *ptr++ = static_cast(values[1] >> 18); + *ptr++ = static_cast(values[1] >> 10); + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 51); + *ptr++ = static_cast(values[2] >> 43); + *ptr++ = static_cast(values[2] >> 35); + *ptr++ = static_cast(values[2] >> 27); + *ptr++ = static_cast(values[2] >> 19); + *ptr++ = static_cast(values[2] >> 11); + *ptr++ = static_cast(values[2] >> 3); + + *ptr = static_cast(values[2] << 5); + *ptr++ |= static_cast(values[3] >> 52); + *ptr++ = static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 53); + *ptr++ = static_cast(values[4] >> 45); + *ptr++ = static_cast(values[4] >> 37); + *ptr++ = static_cast(values[4] >> 29); + *ptr++ = static_cast(values[4] >> 21); + *ptr++ = static_cast(values[4] >> 13); + *ptr++ = static_cast(values[4] >> 5); + + *ptr = static_cast(values[4] << 3); + *ptr++ |= static_cast(values[5] >> 54); + *ptr++ = static_cast(values[5] >> 46); + *ptr++ = static_cast(values[5] >> 38); + *ptr++ = static_cast(values[5] >> 30); + *ptr++ = static_cast(values[5] >> 22); + *ptr++ = static_cast(values[5] >> 14); + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 55); + *ptr++ = static_cast(values[6] >> 47); + *ptr++ = static_cast(values[6] >> 39); + *ptr++ = static_cast(values[6] >> 31); + *ptr++ = static_cast(values[6] >> 23); + *ptr++ = static_cast(values[6] >> 15); + *ptr++ = static_cast(values[6] >> 7); + + *ptr = static_cast(values[6] << 1); + *ptr++ |= static_cast(values[7] >> 56); + *ptr++ = static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits58(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 50); + *ptr++ = static_cast(values[0] >> 42); + *ptr++ = static_cast(values[0] >> 34); + *ptr++ = static_cast(values[0] >> 26); + *ptr++ = static_cast(values[0] >> 18); + *ptr++ = static_cast(values[0] >> 10); + *ptr++ = static_cast(values[0] >> 2); + + *ptr = static_cast(values[0] << 6); + *ptr++ |= static_cast(values[1] >> 52); + *ptr++ = static_cast(values[1] >> 44); + *ptr++ = static_cast(values[1] >> 36); + *ptr++ = static_cast(values[1] >> 28); + *ptr++ = static_cast(values[1] >> 20); + *ptr++ = static_cast(values[1] >> 12); + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 54); + *ptr++ = static_cast(values[2] >> 46); + *ptr++ = static_cast(values[2] >> 38); + *ptr++ = static_cast(values[2] >> 30); + *ptr++ = static_cast(values[2] >> 22); + *ptr++ = static_cast(values[2] >> 14); + *ptr++ = static_cast(values[2] >> 6); + + *ptr = static_cast(values[2] << 2); + *ptr++ |= static_cast(values[3] >> 56); + *ptr++ = static_cast(values[3] >> 48); + *ptr++ = static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 50); + *ptr++ = static_cast(values[4] >> 42); + *ptr++ = static_cast(values[4] >> 34); + *ptr++ = static_cast(values[4] >> 26); + *ptr++ = static_cast(values[4] >> 18); + *ptr++ = static_cast(values[4] >> 10); + *ptr++ = static_cast(values[4] >> 2); + + *ptr = static_cast(values[4] << 6); + *ptr++ |= static_cast(values[5] >> 52); + *ptr++ = static_cast(values[5] >> 44); + *ptr++ = static_cast(values[5] >> 36); + *ptr++ = static_cast(values[5] >> 28); + *ptr++ = static_cast(values[5] >> 20); + *ptr++ = static_cast(values[5] >> 12); + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 54); + *ptr++ = static_cast(values[6] >> 46); + *ptr++ = static_cast(values[6] >> 38); + *ptr++ = static_cast(values[6] >> 30); + *ptr++ = static_cast(values[6] >> 22); + *ptr++ = static_cast(values[6] >> 14); + *ptr++ = static_cast(values[6] >> 6); + + *ptr = static_cast(values[6] << 2); + *ptr++ |= static_cast(values[7] >> 56); + *ptr++ = static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits59(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 51); + *ptr++ = static_cast(values[0] >> 43); + *ptr++ = static_cast(values[0] >> 35); + *ptr++ = static_cast(values[0] >> 27); + *ptr++ = static_cast(values[0] >> 19); + *ptr++ = static_cast(values[0] >> 11); + *ptr++ = static_cast(values[0] >> 3); + + *ptr = static_cast(values[0] << 5); + *ptr++ |= static_cast(values[1] >> 54); + *ptr++ = static_cast(values[1] >> 46); + *ptr++ = static_cast(values[1] >> 38); + *ptr++ = static_cast(values[1] >> 30); + *ptr++ = static_cast(values[1] >> 22); + *ptr++ = static_cast(values[1] >> 14); + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 57); + *ptr++ = static_cast(values[2] >> 49); + *ptr++ = static_cast(values[2] >> 41); + *ptr++ = static_cast(values[2] >> 33); + *ptr++ = static_cast(values[2] >> 25); + *ptr++ = static_cast(values[2] >> 17); + *ptr++ = static_cast(values[2] >> 9); + *ptr++ = static_cast(values[2] >> 1); + + *ptr = static_cast(values[2] << 7); + *ptr++ |= static_cast(values[3] >> 52); + *ptr++ = static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 55); + *ptr++ = static_cast(values[4] >> 47); + *ptr++ = static_cast(values[4] >> 39); + *ptr++ = static_cast(values[4] >> 31); + *ptr++ = static_cast(values[4] >> 23); + *ptr++ = static_cast(values[4] >> 15); + *ptr++ = static_cast(values[4] >> 7); + + *ptr = static_cast(values[4] << 1); + *ptr++ |= static_cast(values[5] >> 58); + *ptr++ = static_cast(values[5] >> 50); + *ptr++ = static_cast(values[5] >> 42); + *ptr++ = static_cast(values[5] >> 34); + *ptr++ = static_cast(values[5] >> 26); + *ptr++ = static_cast(values[5] >> 18); + *ptr++ = static_cast(values[5] >> 10); + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 53); + *ptr++ = static_cast(values[6] >> 45); + *ptr++ = static_cast(values[6] >> 37); + *ptr++ = static_cast(values[6] >> 29); + *ptr++ = static_cast(values[6] >> 21); + *ptr++ = static_cast(values[6] >> 13); + *ptr++ = static_cast(values[6] >> 5); + + *ptr = static_cast(values[6] << 3); + *ptr++ |= static_cast(values[7] >> 56); + *ptr++ = static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits60(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 52); + *ptr++ = static_cast(values[0] >> 44); + *ptr++ = static_cast(values[0] >> 36); + *ptr++ = static_cast(values[0] >> 28); + *ptr++ = static_cast(values[0] >> 20); + *ptr++ = static_cast(values[0] >> 12); + *ptr++ = static_cast(values[0] >> 4); + + *ptr = static_cast(values[0] << 4); + *ptr++ |= static_cast(values[1] >> 56); + *ptr++ = static_cast(values[1] >> 48); + *ptr++ = static_cast(values[1] >> 40); + *ptr++ = static_cast(values[1] >> 32); + *ptr++ = static_cast(values[1] >> 24); + *ptr++ = static_cast(values[1] >> 16); + *ptr++ = static_cast(values[1] >> 8); + *ptr++ = static_cast(values[1]); + + *ptr++ = static_cast(values[2] >> 52); + *ptr++ = static_cast(values[2] >> 44); + *ptr++ = static_cast(values[2] >> 36); + *ptr++ = static_cast(values[2] >> 28); + *ptr++ = static_cast(values[2] >> 20); + *ptr++ = static_cast(values[2] >> 12); + *ptr++ = static_cast(values[2] >> 4); + + *ptr = static_cast(values[2] << 4); + *ptr++ |= static_cast(values[3] >> 56); + *ptr++ = static_cast(values[3] >> 48); + *ptr++ = static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 52); + *ptr++ = static_cast(values[4] >> 44); + *ptr++ = static_cast(values[4] >> 36); + *ptr++ = static_cast(values[4] >> 28); + *ptr++ = static_cast(values[4] >> 20); + *ptr++ = static_cast(values[4] >> 12); + *ptr++ = static_cast(values[4] >> 4); + + *ptr = static_cast(values[4] << 4); + *ptr++ |= static_cast(values[5] >> 56); + *ptr++ = static_cast(values[5] >> 48); + *ptr++ = static_cast(values[5] >> 40); + *ptr++ = static_cast(values[5] >> 32); + *ptr++ = static_cast(values[5] >> 24); + *ptr++ = static_cast(values[5] >> 16); + *ptr++ = static_cast(values[5] >> 8); + *ptr++ = static_cast(values[5]); + + *ptr++ = static_cast(values[6] >> 52); + *ptr++ = static_cast(values[6] >> 44); + *ptr++ = static_cast(values[6] >> 36); + *ptr++ = static_cast(values[6] >> 28); + *ptr++ = static_cast(values[6] >> 20); + *ptr++ = static_cast(values[6] >> 12); + *ptr++ = static_cast(values[6] >> 4); + + *ptr = static_cast(values[6] << 4); + *ptr++ |= static_cast(values[7] >> 56); + *ptr++ = static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits61(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 53); + *ptr++ = static_cast(values[0] >> 45); + *ptr++ = static_cast(values[0] >> 37); + *ptr++ = static_cast(values[0] >> 29); + *ptr++ = static_cast(values[0] >> 21); + *ptr++ = static_cast(values[0] >> 13); + *ptr++ = static_cast(values[0] >> 5); + + *ptr = static_cast(values[0] << 3); + *ptr++ |= static_cast(values[1] >> 58); + *ptr++ = static_cast(values[1] >> 50); + *ptr++ = static_cast(values[1] >> 42); + *ptr++ = static_cast(values[1] >> 34); + *ptr++ = static_cast(values[1] >> 26); + *ptr++ = static_cast(values[1] >> 18); + *ptr++ = static_cast(values[1] >> 10); + *ptr++ = static_cast(values[1] >> 2); + + *ptr = static_cast(values[1] << 6); + *ptr++ |= static_cast(values[2] >> 55); + *ptr++ = static_cast(values[2] >> 47); + *ptr++ = static_cast(values[2] >> 39); + *ptr++ = static_cast(values[2] >> 31); + *ptr++ = static_cast(values[2] >> 23); + *ptr++ = static_cast(values[2] >> 15); + *ptr++ = static_cast(values[2] >> 7); + + *ptr = static_cast(values[2] << 1); + *ptr++ |= static_cast(values[3] >> 60); + *ptr++ = static_cast(values[3] >> 52); + *ptr++ = static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 57); + *ptr++ = static_cast(values[4] >> 49); + *ptr++ = static_cast(values[4] >> 41); + *ptr++ = static_cast(values[4] >> 33); + *ptr++ = static_cast(values[4] >> 25); + *ptr++ = static_cast(values[4] >> 17); + *ptr++ = static_cast(values[4] >> 9); + *ptr++ = static_cast(values[4] >> 1); + + *ptr = static_cast(values[4] << 7); + *ptr++ |= static_cast(values[5] >> 54); + *ptr++ = static_cast(values[5] >> 46); + *ptr++ = static_cast(values[5] >> 38); + *ptr++ = static_cast(values[5] >> 30); + *ptr++ = static_cast(values[5] >> 22); + *ptr++ = static_cast(values[5] >> 14); + *ptr++ = static_cast(values[5] >> 6); + + *ptr = static_cast(values[5] << 2); + *ptr++ |= static_cast(values[6] >> 59); + *ptr++ = static_cast(values[6] >> 51); + *ptr++ = static_cast(values[6] >> 43); + *ptr++ = static_cast(values[6] >> 35); + *ptr++ = static_cast(values[6] >> 27); + *ptr++ = static_cast(values[6] >> 19); + *ptr++ = static_cast(values[6] >> 11); + *ptr++ = static_cast(values[6] >> 3); + + *ptr = static_cast(values[6] << 5); + *ptr++ |= static_cast(values[7] >> 56); + *ptr++ = static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits62(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 54); + *ptr++ = static_cast(values[0] >> 46); + *ptr++ = static_cast(values[0] >> 38); + *ptr++ = static_cast(values[0] >> 30); + *ptr++ = static_cast(values[0] >> 22); + *ptr++ = static_cast(values[0] >> 14); + *ptr++ = static_cast(values[0] >> 6); + + *ptr = static_cast(values[0] << 2); + *ptr++ |= static_cast(values[1] >> 60); + *ptr++ = static_cast(values[1] >> 52); + *ptr++ = static_cast(values[1] >> 44); + *ptr++ = static_cast(values[1] >> 36); + *ptr++ = static_cast(values[1] >> 28); + *ptr++ = static_cast(values[1] >> 20); + *ptr++ = static_cast(values[1] >> 12); + *ptr++ = static_cast(values[1] >> 4); + + *ptr = static_cast(values[1] << 4); + *ptr++ |= static_cast(values[2] >> 58); + *ptr++ = static_cast(values[2] >> 50); + *ptr++ = static_cast(values[2] >> 42); + *ptr++ = static_cast(values[2] >> 34); + *ptr++ = static_cast(values[2] >> 26); + *ptr++ = static_cast(values[2] >> 18); + *ptr++ = static_cast(values[2] >> 10); + *ptr++ = static_cast(values[2] >> 2); + + *ptr = static_cast(values[2] << 6); + *ptr++ |= static_cast(values[3] >> 56); + *ptr++ = static_cast(values[3] >> 48); + *ptr++ = static_cast(values[3] >> 40); + *ptr++ = static_cast(values[3] >> 32); + *ptr++ = static_cast(values[3] >> 24); + *ptr++ = static_cast(values[3] >> 16); + *ptr++ = static_cast(values[3] >> 8); + *ptr++ = static_cast(values[3]); + + *ptr++ = static_cast(values[4] >> 54); + *ptr++ = static_cast(values[4] >> 46); + *ptr++ = static_cast(values[4] >> 38); + *ptr++ = static_cast(values[4] >> 30); + *ptr++ = static_cast(values[4] >> 22); + *ptr++ = static_cast(values[4] >> 14); + *ptr++ = static_cast(values[4] >> 6); + + *ptr = static_cast(values[4] << 2); + *ptr++ |= static_cast(values[5] >> 60); + *ptr++ = static_cast(values[5] >> 52); + *ptr++ = static_cast(values[5] >> 44); + *ptr++ = static_cast(values[5] >> 36); + *ptr++ = static_cast(values[5] >> 28); + *ptr++ = static_cast(values[5] >> 20); + *ptr++ = static_cast(values[5] >> 12); + *ptr++ = static_cast(values[5] >> 4); + + *ptr = static_cast(values[5] << 4); + *ptr++ |= static_cast(values[6] >> 58); + *ptr++ = static_cast(values[6] >> 50); + *ptr++ = static_cast(values[6] >> 42); + *ptr++ = static_cast(values[6] >> 34); + *ptr++ = static_cast(values[6] >> 26); + *ptr++ = static_cast(values[6] >> 18); + *ptr++ = static_cast(values[6] >> 10); + *ptr++ = static_cast(values[6] >> 2); + + *ptr = static_cast(values[6] << 6); + *ptr++ |= static_cast(values[7] >> 56); + *ptr++ = static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void packBits63(const uint64_t* values, uint8_t* ptr) { + *ptr++ = static_cast(values[0] >> 55); + *ptr++ = static_cast(values[0] >> 47); + *ptr++ = static_cast(values[0] >> 39); + *ptr++ = static_cast(values[0] >> 31); + *ptr++ = static_cast(values[0] >> 23); + *ptr++ = static_cast(values[0] >> 15); + *ptr++ = static_cast(values[0] >> 7); + + *ptr = static_cast(values[0] << 1); + *ptr++ |= static_cast(values[1] >> 62); + *ptr++ = static_cast(values[1] >> 54); + *ptr++ = static_cast(values[1] >> 46); + *ptr++ = static_cast(values[1] >> 38); + *ptr++ = static_cast(values[1] >> 30); + *ptr++ = static_cast(values[1] >> 22); + *ptr++ = static_cast(values[1] >> 14); + *ptr++ = static_cast(values[1] >> 6); + + *ptr = static_cast(values[1] << 2); + *ptr++ |= static_cast(values[2] >> 61); + *ptr++ = static_cast(values[2] >> 53); + *ptr++ = static_cast(values[2] >> 45); + *ptr++ = static_cast(values[2] >> 37); + *ptr++ = static_cast(values[2] >> 29); + *ptr++ = static_cast(values[2] >> 21); + *ptr++ = static_cast(values[2] >> 13); + *ptr++ = static_cast(values[2] >> 5); + + *ptr = static_cast(values[2] << 3); + *ptr++ |= static_cast(values[3] >> 60); + *ptr++ = static_cast(values[3] >> 52); + *ptr++ = static_cast(values[3] >> 44); + *ptr++ = static_cast(values[3] >> 36); + *ptr++ = static_cast(values[3] >> 28); + *ptr++ = static_cast(values[3] >> 20); + *ptr++ = static_cast(values[3] >> 12); + *ptr++ = static_cast(values[3] >> 4); + + *ptr = static_cast(values[3] << 4); + *ptr++ |= static_cast(values[4] >> 59); + *ptr++ = static_cast(values[4] >> 51); + *ptr++ = static_cast(values[4] >> 43); + *ptr++ = static_cast(values[4] >> 35); + *ptr++ = static_cast(values[4] >> 27); + *ptr++ = static_cast(values[4] >> 19); + *ptr++ = static_cast(values[4] >> 11); + *ptr++ = static_cast(values[4] >> 3); + + *ptr = static_cast(values[4] << 5); + *ptr++ |= static_cast(values[5] >> 58); + *ptr++ = static_cast(values[5] >> 50); + *ptr++ = static_cast(values[5] >> 42); + *ptr++ = static_cast(values[5] >> 34); + *ptr++ = static_cast(values[5] >> 26); + *ptr++ = static_cast(values[5] >> 18); + *ptr++ = static_cast(values[5] >> 10); + *ptr++ = static_cast(values[5] >> 2); + + *ptr = static_cast(values[5] << 6); + *ptr++ |= static_cast(values[6] >> 57); + *ptr++ = static_cast(values[6] >> 49); + *ptr++ = static_cast(values[6] >> 41); + *ptr++ = static_cast(values[6] >> 33); + *ptr++ = static_cast(values[6] >> 25); + *ptr++ = static_cast(values[6] >> 17); + *ptr++ = static_cast(values[6] >> 9); + *ptr++ = static_cast(values[6] >> 1); + + *ptr = static_cast(values[6] << 7); + *ptr++ |= static_cast(values[7] >> 56); + *ptr++ = static_cast(values[7] >> 48); + *ptr++ = static_cast(values[7] >> 40); + *ptr++ = static_cast(values[7] >> 32); + *ptr++ = static_cast(values[7] >> 24); + *ptr++ = static_cast(values[7] >> 16); + *ptr++ = static_cast(values[7] >> 8); + *ptr = static_cast(values[7]); +} + +static inline void unpackBits1(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr >> 7; + values[1] = (*ptr >> 6) & 1; + values[2] = (*ptr >> 5) & 1; + values[3] = (*ptr >> 4) & 1; + values[4] = (*ptr >> 3) & 1; + values[5] = (*ptr >> 2) & 1; + values[6] = (*ptr >> 1) & 1; + values[7] = *ptr & 1; +} + +static inline void unpackBits2(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr >> 6; + values[1] = (*ptr >> 4) & 3; + values[2] = (*ptr >> 2) & 3; + values[3] = *ptr++ & 3; + values[4] = *ptr >> 6; + values[5] = (*ptr >> 4) & 3; + values[6] = (*ptr >> 2) & 3; + values[7] = *ptr & 3; +} + +static inline void unpackBits3(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr >> 5; + values[1] = (*ptr >> 2) & 7; + values[2] = (*ptr++ & 3) << 1; + values[2] |= *ptr >> 7; + values[3] = (*ptr >> 4) & 7; + values[4] = (*ptr >> 1) & 7; + values[5] = (*ptr++ & 1) << 2; + values[5] |= *ptr >> 6; + values[6] = (*ptr >> 3) & 7; + values[7] = *ptr & 7; +} + +static inline void unpackBits4(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr >> 4; + values[1] = *ptr++ & 0xf; + values[2] = *ptr >> 4; + values[3] = *ptr++ & 0xf; + values[4] = *ptr >> 4; + values[5] = *ptr++ & 0xf; + values[6] = *ptr >> 4; + values[7] = *ptr & 0xf; +} + +static inline void unpackBits5(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr >> 3; + + values[1] = (*ptr++ & 7) << 2; + values[1] |= *ptr >> 6; + + values[2] = (*ptr >> 1) & 0x1f; + + values[3] = (*ptr++ & 1) << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 1; + values[4] |= *ptr >> 7; + + values[5] = (*ptr >> 2) & 0x1f; + + values[6] = (*ptr++ & 3) << 3; + values[6] |= *ptr >> 5; + + values[7] = *ptr & 0x1f; +} + +static inline void unpackBits6(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr >> 2; + + values[1] = (*ptr++ & 3) << 4; + values[1] |= *ptr >> 4; + + values[2] = (*ptr++ & 0xf) << 2; + values[2] |= *ptr >> 6; + + values[3] = *ptr++ & 0x3f; + + values[4] = *ptr >> 2; + + values[5] = (*ptr++ & 3) << 4; + values[5] |= *ptr >> 4; + + values[6] = (*ptr++ & 0xf) << 2; + values[6] |= *ptr >> 6; + + values[7] = *ptr & 0x3f; +} + +static inline void unpackBits7(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr >> 1; + + values[1] = (*ptr++ & 1) << 6; + values[1] |= *ptr >> 2; + + values[2] = (*ptr++ & 3) << 5; + values[2] |= *ptr >> 3; + + values[3] = (*ptr++ & 7) << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 3; + values[4] |= *ptr >> 5; + + values[5] = (*ptr++ & 0x1f) << 2; + values[5] |= *ptr >> 6; + + values[6] = (*ptr++ & 0x3f) << 1; + values[6] |= *ptr >> 7; + + values[7] = *ptr & 0x7f; +} + +static inline void unpackBits8(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++; + values[1] = *ptr++; + values[2] = *ptr++; + values[3] = *ptr++; + values[4] = *ptr++; + values[5] = *ptr++; + values[6] = *ptr++; + values[7] = *ptr; +} + +static inline void unpackBits9(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 1; + values[0] |= *ptr >> 7; + + values[1] = (*ptr++ & 0x7f) << 2; + values[1] |= *ptr >> 6; + + values[2] = (*ptr++ & 0x3f) << 3; + values[2] |= *ptr >> 5; + + values[3] = (*ptr++ & 0x1f) << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 5; + values[4] |= *ptr >> 3; + + values[5] = (*ptr++ & 7) << 6; + values[5] |= *ptr >> 2; + + values[6] = (*ptr++ & 3) << 7; + values[6] |= *ptr >> 1; + + values[7] = (*ptr++ & 1) << 8; + values[7] |= *ptr; +} + +static inline void unpackBits10(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 2; + values[0] |= *ptr >> 6; + + values[1] = (*ptr++ & 0x3f) << 4; + values[1] |= *ptr >> 4; + + values[2] = (*ptr++ & 0xf) << 6; + values[2] |= *ptr >> 2; + + values[3] = (*ptr++ & 3) << 8; + values[3] |= *ptr++; + + values[4] = *ptr++ << 2; + values[4] |= *ptr >> 6; + + values[5] = (*ptr++ & 0x3f) << 4; + values[5] |= *ptr >> 4; + + values[6] = (*ptr++ & 0xf) << 6; + values[6] |= *ptr >> 2; + + values[7] = (*ptr++ & 3) << 8; + values[7] |= *ptr; +} + +static inline void unpackBits11(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 3; + values[0] |= *ptr >> 5; + + values[1] = (*ptr++ & 0x1f) << 6; + values[1] |= *ptr >> 2; + + values[2] = (*ptr++ & 3) << 9; + values[2] |= *ptr++ << 1; + values[2] |= *ptr >> 7; + + values[3] = (*ptr++ & 0x7f) << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 7; + values[4] |= *ptr >> 1; + + values[5] = (*ptr++ & 1) << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = (*ptr++ & 0x3f) << 5; + values[6] |= *ptr >> 3; + + values[7] = (*ptr++ & 7) << 8; + values[7] |= *ptr; +} + +static inline void unpackBits12(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 4; + values[0] |= *ptr >> 4; + + values[1] = (*ptr++ & 0xf) << 8; + values[1] |= *ptr++; + + values[2] = *ptr++ << 4; + values[2] |= *ptr >> 4; + + values[3] = (*ptr++ & 0xf) << 8; + values[3] |= *ptr++; + + values[4] = *ptr++ << 4; + values[4] |= *ptr >> 4; + + values[5] = (*ptr++ & 0xf) << 8; + values[5] |= *ptr++; + + values[6] = *ptr++ << 4; + values[6] |= *ptr >> 4; + + values[7] = (*ptr++ & 0xf) << 8; + values[7] |= *ptr; +} + +static inline void unpackBits13(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 5; + values[0] |= *ptr >> 3; + + values[1] = (*ptr++ & 7) << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = (*ptr++ & 0x3f) << 7; + values[2] |= *ptr >> 1; + + values[3] = (*ptr++ & 1) << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 9; + values[4] |= *ptr++ << 1; + values[4] |= *ptr >> 7; + + values[5] = (*ptr++ & 0x7f) << 6; + values[5] |= *ptr >> 2; + + values[6] = (*ptr++ & 3) << 11; + values[6] |= *ptr++ << 3; + values[6] |= *ptr >> 5; + + values[7] = (*ptr++ & 0x1f) << 8; + values[7] |= *ptr; +} + +static inline void unpackBits14(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 6; + values[0] |= *ptr >> 2; + + values[1] = (*ptr++ & 3) << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = (*ptr++ & 0xf) << 10; + values[2] |= *ptr++ << 2; + values[2] |= *ptr >> 6; + + values[3] = (*ptr++ & 0x3f) << 8; + values[3] |= *ptr++; + + values[4] = *ptr++ << 6; + values[4] |= *ptr >> 2; + + values[5] = (*ptr++ & 3) << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = (*ptr++ & 0xf) << 10; + values[6] |= *ptr++ << 2; + values[6] |= *ptr >> 6; + + values[7] = (*ptr++ & 0x3f) << 8; + values[7] |= *ptr; +} + +static inline void unpackBits15(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 7; + values[0] |= *ptr >> 1; + + values[1] = (*ptr++ & 1) << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = (*ptr++ & 3) << 13; + values[2] |= *ptr++ << 5; + values[2] |= *ptr >> 3; + + values[3] = (*ptr++ & 7) << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 11; + values[4] |= *ptr++ << 3; + values[4] |= *ptr >> 5; + + values[5] = (*ptr++ & 0x1f) << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = (*ptr++ & 0x3f) << 9; + values[6] |= *ptr++ << 1; + values[6] |= *ptr >> 7; + + values[7] = (*ptr++ & 0x7f) << 8; + values[7] |= *ptr; +} + +static inline void unpackBits16(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 8; + values[0] |= *ptr++; + values[1] = *ptr++ << 8; + values[1] |= *ptr++; + values[2] = *ptr++ << 8; + values[2] |= *ptr++; + values[3] = *ptr++ << 8; + values[3] |= *ptr++; + values[4] = *ptr++ << 8; + values[4] |= *ptr++; + values[5] = *ptr++ << 8; + values[5] |= *ptr++; + values[6] = *ptr++ << 8; + values[6] |= *ptr++; + values[7] = *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits17(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 9; + values[0] |= *ptr++ << 1; + values[0] |= *ptr >> 7; + + values[1] = (*ptr++ & 0x7f) << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = (*ptr++ & 0x3f) << 11; + values[2] |= *ptr++ << 3; + values[2] |= *ptr >> 5; + + values[3] = (*ptr++ & 0x1f) << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 13; + values[4] |= *ptr++ << 5; + values[4] |= *ptr >> 3; + + values[5] = (*ptr++ & 7) << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = (*ptr++ & 3) << 15; + values[6] |= *ptr++ << 7; + values[6] |= *ptr >> 1; + + values[7] = (*ptr++ & 1) << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits18(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 10; + values[0] |= *ptr++ << 2; + values[0] |= *ptr >> 6; + + values[1] = (*ptr++ & 0x3f) << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = (*ptr++ & 0xf) << 14; + values[2] |= *ptr++ << 6; + values[2] |= *ptr >> 2; + + values[3] = (*ptr++ & 3) << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = *ptr++ << 10; + values[4] |= *ptr++ << 2; + values[4] |= *ptr >> 6; + + values[5] = (*ptr++ & 0x3f) << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = (*ptr++ & 0xf) << 14; + values[6] |= *ptr++ << 6; + values[6] |= *ptr >> 2; + + values[7] = (*ptr++ & 3) << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits19(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 11; + values[0] |= *ptr++ << 3; + values[0] |= *ptr >> 5; + + values[1] = (*ptr++ & 0x1f) << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = (*ptr++ & 3) << 17; + values[2] |= *ptr++ << 9; + values[2] |= *ptr++ << 1; + values[2] |= *ptr >> 7; + + values[3] = (*ptr++ & 0x7f) << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 15; + values[4] |= *ptr++ << 7; + values[4] |= *ptr >> 1; + + values[5] = (*ptr++ & 1) << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = (*ptr++ & 0x3f) << 13; + values[6] |= *ptr++ << 5; + values[6] |= *ptr >> 3; + + values[7] = (*ptr++ & 7) << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits20(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 12; + values[0] |= *ptr++ << 4; + values[0] |= *ptr >> 4; + + values[1] = (*ptr++ & 0xf) << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + + values[2] = *ptr++ << 12; + values[2] |= *ptr++ << 4; + values[2] |= *ptr >> 4; + + values[3] = (*ptr++ & 0xf) << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = *ptr++ << 12; + values[4] |= *ptr++ << 4; + values[4] |= *ptr >> 4; + + values[5] = (*ptr++ & 0xf) << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + + values[6] = *ptr++ << 12; + values[6] |= *ptr++ << 4; + values[6] |= *ptr >> 4; + + values[7] = (*ptr++ & 0xf) << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits21(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 13; + values[0] |= *ptr++ << 5; + values[0] |= *ptr >> 3; + + values[1] = (*ptr++ & 7) << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = (*ptr++ & 0x3f) << 15; + values[2] |= *ptr++ << 7; + values[2] |= *ptr >> 1; + + values[3] = (*ptr++ & 1) << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 17; + values[4] |= *ptr++ << 9; + values[4] |= *ptr++ << 1; + values[4] |= *ptr >> 7; + + values[5] = (*ptr++ & 0x7f) << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = (*ptr++ & 3) << 19; + values[6] |= *ptr++ << 11; + values[6] |= *ptr++ << 3; + values[6] |= *ptr >> 5; + + values[7] = (*ptr++ & 0x1f) << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits22(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 14; + values[0] |= *ptr++ << 6; + values[0] |= *ptr >> 2; + + values[1] = (*ptr++ & 3) << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = (*ptr++ & 0xf) << 18; + values[2] |= *ptr++ << 10; + values[2] |= *ptr++ << 2; + values[2] |= *ptr >> 6; + + values[3] = (*ptr++ & 0x3f) << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = *ptr++ << 14; + values[4] |= *ptr++ << 6; + values[4] |= *ptr >> 2; + + values[5] = (*ptr++ & 3) << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = (*ptr++ & 0xf) << 18; + values[6] |= *ptr++ << 10; + values[6] |= *ptr++ << 2; + values[6] |= *ptr >> 6; + + values[7] = (*ptr++ & 0x3f) << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits23(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 15; + values[0] |= *ptr++ << 7; + values[0] |= *ptr >> 1; + + values[1] = (*ptr++ & 1) << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = (*ptr++ & 3) << 21; + values[2] |= *ptr++ << 13; + values[2] |= *ptr++ << 5; + values[2] |= *ptr >> 3; + + values[3] = (*ptr++ & 7) << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 19; + values[4] |= *ptr++ << 11; + values[4] |= *ptr++ << 3; + values[4] |= *ptr >> 5; + + values[5] = (*ptr++ & 0x1f) << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = (*ptr++ & 0x3f) << 17; + values[6] |= *ptr++ << 9; + values[6] |= *ptr++ << 1; + values[6] |= *ptr >> 7; + + values[7] = (*ptr++ & 0x7f) << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits24(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 16; + values[0] |= *ptr++ << 8; + values[0] |= *ptr++; + values[1] = *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + values[2] = *ptr++ << 16; + values[2] |= *ptr++ << 8; + values[2] |= *ptr++; + values[3] = *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + values[4] = *ptr++ << 16; + values[4] |= *ptr++ << 8; + values[4] |= *ptr++; + values[5] = *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + values[6] = *ptr++ << 16; + values[6] |= *ptr++ << 8; + values[6] |= *ptr++; + values[7] = *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits25(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 17; + values[0] |= *ptr++ << 9; + values[0] |= *ptr++ << 1; + values[0] |= *ptr >> 7; + + values[1] = (*ptr++ & 0x7f) << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = (*ptr++ & 0x3f) << 19; + values[2] |= *ptr++ << 11; + values[2] |= *ptr++ << 3; + values[2] |= *ptr >> 5; + + values[3] = (*ptr++ & 0x1f) << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 21; + values[4] |= *ptr++ << 13; + values[4] |= *ptr++ << 5; + values[4] |= *ptr >> 3; + + values[5] = (*ptr++ & 7) << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = (*ptr++ & 3) << 23; + values[6] |= *ptr++ << 15; + values[6] |= *ptr++ << 7; + values[6] |= *ptr >> 1; + + values[7] = static_cast(*ptr++ & 1) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits26(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 18; + values[0] |= *ptr++ << 10; + values[0] |= *ptr++ << 2; + values[0] |= *ptr >> 6; + + values[1] = (*ptr++ & 0x3f) << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = (*ptr++ & 0xf) << 22; + values[2] |= *ptr++ << 14; + values[2] |= *ptr++ << 6; + values[2] |= *ptr >> 2; + + values[3] = static_cast(*ptr++ & 3) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = *ptr++ << 18; + values[4] |= *ptr++ << 10; + values[4] |= *ptr++ << 2; + values[4] |= *ptr >> 6; + + values[5] = (*ptr++ & 0x3f) << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = (*ptr++ & 0xf) << 22; + values[6] |= *ptr++ << 14; + values[6] |= *ptr++ << 6; + values[6] |= *ptr >> 2; + + values[7] = static_cast(*ptr++ & 3) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits27(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 19; + values[0] |= *ptr++ << 11; + values[0] |= *ptr++ << 3; + values[0] |= *ptr >> 5; + + values[1] = (*ptr++ & 0x1f) << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 25; + values[2] |= *ptr++ << 17; + values[2] |= *ptr++ << 9; + values[2] |= *ptr++ << 1; + values[2] |= *ptr >> 7; + + values[3] = (*ptr++ & 0x7f) << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = (*ptr++ & 0xf) << 23; + values[4] |= *ptr++ << 15; + values[4] |= *ptr++ << 7; + values[4] |= *ptr >> 1; + + values[5] = static_cast(*ptr++ & 1) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = (*ptr++ & 0x3f) << 21; + values[6] |= *ptr++ << 13; + values[6] |= *ptr++ << 5; + values[6] |= *ptr >> 3; + + values[7] = static_cast(*ptr++ & 7) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits28(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 20; + values[0] |= *ptr++ << 12; + values[0] |= *ptr++ << 4; + values[0] |= *ptr >> 4; + + values[1] = static_cast(*ptr++ & 0xf) << 24; + values[1] |= *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + + values[2] = *ptr++ << 20; + values[2] |= *ptr++ << 12; + values[2] |= *ptr++ << 4; + values[2] |= *ptr >> 4; + + values[3] = static_cast(*ptr++ & 0xf) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = *ptr++ << 20; + values[4] |= *ptr++ << 12; + values[4] |= *ptr++ << 4; + values[4] |= *ptr >> 4; + + values[5] = static_cast(*ptr++ & 0xf) << 24; + values[5] |= *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + + values[6] = *ptr++ << 20; + values[6] |= *ptr++ << 12; + values[6] |= *ptr++ << 4; + values[6] |= *ptr >> 4; + + values[7] = static_cast(*ptr++ & 0xf) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits29(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 21; + values[0] |= *ptr++ << 13; + values[0] |= *ptr++ << 5; + values[0] |= *ptr >> 3; + + values[1] = static_cast(*ptr++ & 7) << 26; + values[1] |= *ptr++ << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = (*ptr++ & 0x3f) << 23; + values[2] |= *ptr++ << 15; + values[2] |= *ptr++ << 7; + values[2] |= *ptr >> 1; + + values[3] = static_cast(*ptr++ & 1) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 25; + values[4] |= *ptr++ << 17; + values[4] |= *ptr++ << 9; + values[4] |= *ptr++ << 1; + values[4] |= *ptr >> 7; + + values[5] = (*ptr++ & 0x7f) << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = static_cast(*ptr++ & 3) << 27; + values[6] |= *ptr++ << 19; + values[6] |= *ptr++ << 11; + values[6] |= *ptr++ << 3; + values[6] |= *ptr >> 5; + + values[7] = static_cast(*ptr++ & 0x1f) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits30(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 22; + values[0] |= *ptr++ << 14; + values[0] |= *ptr++ << 6; + values[0] |= *ptr >> 2; + + values[1] = static_cast(*ptr++ & 3) << 28; + values[1] |= *ptr++ << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = static_cast(*ptr++ & 0xf) << 26; + values[2] |= *ptr++ << 18; + values[2] |= *ptr++ << 10; + values[2] |= *ptr++ << 2; + values[2] |= *ptr >> 6; + + values[3] = static_cast(*ptr++ & 0x3f) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = *ptr++ << 22; + values[4] |= *ptr++ << 14; + values[4] |= *ptr++ << 6; + values[4] |= *ptr >> 2; + + values[5] = static_cast(*ptr++ & 3) << 28; + values[5] |= *ptr++ << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = static_cast(*ptr++ & 0xf) << 26; + values[6] |= *ptr++ << 18; + values[6] |= *ptr++ << 10; + values[6] |= *ptr++ << 2; + values[6] |= *ptr >> 6; + + values[7] = static_cast(*ptr++ & 0x3f) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits31(uint64_t* values, const uint8_t* ptr) { + values[0] = *ptr++ << 23; + values[0] |= *ptr++ << 15; + values[0] |= *ptr++ << 7; + values[0] |= *ptr >> 1; + + values[1] = static_cast(*ptr++ & 1) << 30; + values[1] |= *ptr++ << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 29; + values[2] |= *ptr++ << 21; + values[2] |= *ptr++ << 13; + values[2] |= *ptr++ << 5; + values[2] |= *ptr >> 3; + + values[3] = static_cast(*ptr++ & 7) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 27; + values[4] |= *ptr++ << 19; + values[4] |= *ptr++ << 11; + values[4] |= *ptr++ << 3; + values[4] |= *ptr >> 5; + + values[5] = static_cast(*ptr++ & 0x1f) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = static_cast(*ptr++ & 0x3f) << 25; + values[6] |= *ptr++ << 17; + values[6] |= *ptr++ << 9; + values[6] |= *ptr++ << 1; + values[6] |= *ptr >> 7; + + values[7] = static_cast(*ptr++ & 0x7f) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits32(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 24; + values[0] |= *ptr++ << 16; + values[0] |= *ptr++ << 8; + values[0] |= *ptr++; + values[1] = static_cast(*ptr++) << 24; + values[1] |= *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + values[2] = static_cast(*ptr++) << 24; + values[2] |= *ptr++ << 16; + values[2] |= *ptr++ << 8; + values[2] |= *ptr++; + values[3] = static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + values[4] = static_cast(*ptr++) << 24; + values[4] |= *ptr++ << 16; + values[4] |= *ptr++ << 8; + values[4] |= *ptr++; + values[5] = static_cast(*ptr++) << 24; + values[5] |= *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + values[6] = static_cast(*ptr++) << 24; + values[6] |= *ptr++ << 16; + values[6] |= *ptr++ << 8; + values[6] |= *ptr++; + values[7] = static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits33(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 25; + values[0] |= *ptr++ << 17; + values[0] |= *ptr++ << 9; + values[0] |= *ptr++ << 1; + values[0] |= *ptr >> 7; + + values[1] = static_cast(*ptr++ & 0x7f) << 26; + values[1] |= *ptr++ << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = static_cast(*ptr++ & 0x3f) << 27; + values[2] |= *ptr++ << 19; + values[2] |= *ptr++ << 11; + values[2] |= *ptr++ << 3; + values[2] |= *ptr >> 5; + + values[3] = static_cast(*ptr++ & 0x1f) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 29; + values[4] |= *ptr++ << 21; + values[4] |= *ptr++ << 13; + values[4] |= *ptr++ << 5; + values[4] |= *ptr >> 3; + + values[5] = static_cast(*ptr++ & 7) << 30; + values[5] |= *ptr++ << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = static_cast(*ptr++ & 3) << 31; + values[6] |= *ptr++ << 23; + values[6] |= *ptr++ << 15; + values[6] |= *ptr++ << 7; + values[6] |= *ptr >> 1; + + values[7] = static_cast(*ptr++ & 1) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits34(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 26; + values[0] |= *ptr++ << 18; + values[0] |= *ptr++ << 10; + values[0] |= *ptr++ << 2; + values[0] |= *ptr >> 6; + + values[1] = static_cast(*ptr++ & 0x3f) << 28; + values[1] |= *ptr++ << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = static_cast(*ptr++ & 0xf) << 30; + values[2] |= *ptr++ << 22; + values[2] |= *ptr++ << 14; + values[2] |= *ptr++ << 6; + values[2] |= *ptr >> 2; + + values[3] = static_cast(*ptr++ & 3) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 26; + values[4] |= *ptr++ << 18; + values[4] |= *ptr++ << 10; + values[4] |= *ptr++ << 2; + values[4] |= *ptr >> 6; + + values[5] = static_cast(*ptr++ & 0x3f) << 28; + values[5] |= *ptr++ << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = static_cast(*ptr++ & 0xf) << 30; + values[6] |= *ptr++ << 22; + values[6] |= *ptr++ << 14; + values[6] |= *ptr++ << 6; + values[6] |= *ptr >> 2; + + values[7] = static_cast(*ptr++ & 3) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr++; +} + +static inline void unpackBits35(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 27; + values[0] |= *ptr++ << 19; + values[0] |= *ptr++ << 11; + values[0] |= *ptr++ << 3; + values[0] |= *ptr >> 5; + + values[1] = static_cast(*ptr++ & 0x1f) << 30; + values[1] |= *ptr++ << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 33; + values[2] |= static_cast(*ptr++) << 25; + values[2] |= *ptr++ << 17; + values[2] |= *ptr++ << 9; + values[2] |= *ptr++ << 1; + values[2] |= *ptr >> 7; + + values[3] = static_cast(*ptr++ & 0x7f) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 31; + values[4] |= *ptr++ << 23; + values[4] |= *ptr++ << 15; + values[4] |= *ptr++ << 7; + values[4] |= *ptr >> 1; + + values[5] = static_cast(*ptr++ & 1) << 34; + values[5] |= static_cast(*ptr++) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = static_cast(*ptr++ & 0x3f) << 29; + values[6] |= *ptr++ << 21; + values[6] |= *ptr++ << 13; + values[6] |= *ptr++ << 5; + values[6] |= *ptr >> 3; + + values[7] = static_cast(*ptr++ & 7) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits36(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 28; + values[0] |= *ptr++ << 20; + values[0] |= *ptr++ << 12; + values[0] |= *ptr++ << 4; + values[0] |= *ptr >> 4; + + values[1] = static_cast(*ptr++ & 0xf) << 32; + values[1] |= static_cast(*ptr++) << 24; + values[1] |= *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + + values[2] = static_cast(*ptr++) << 28; + values[2] |= *ptr++ << 20; + values[2] |= *ptr++ << 12; + values[2] |= *ptr++ << 4; + values[2] |= *ptr >> 4; + + values[3] = static_cast(*ptr++ & 0xf) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 28; + values[4] |= *ptr++ << 20; + values[4] |= *ptr++ << 12; + values[4] |= *ptr++ << 4; + values[4] |= *ptr >> 4; + + values[5] = static_cast(*ptr++ & 0xf) << 32; + values[5] |= static_cast(*ptr++) << 24; + values[5] |= *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + + values[6] = static_cast(*ptr++) << 28; + values[6] |= *ptr++ << 20; + values[6] |= *ptr++ << 12; + values[6] |= *ptr++ << 4; + values[6] |= *ptr >> 4; + + values[7] = static_cast(*ptr++ & 0xf) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits37(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 29; + values[0] |= *ptr++ << 21; + values[0] |= *ptr++ << 13; + values[0] |= *ptr++ << 5; + values[0] |= *ptr >> 3; + + values[1] = static_cast(*ptr++ & 7) << 34; + values[1] |= static_cast(*ptr++) << 26; + values[1] |= *ptr++ << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = static_cast(*ptr++ & 0x3f) << 31; + values[2] |= static_cast(*ptr++) << 23; + values[2] |= *ptr++ << 15; + values[2] |= *ptr++ << 7; + values[2] |= *ptr >> 1; + + values[3] = static_cast(*ptr++ & 1) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 33; + values[4] |= static_cast(*ptr++) << 25; + values[4] |= *ptr++ << 17; + values[4] |= *ptr++ << 9; + values[4] |= *ptr++ << 1; + values[4] |= *ptr >> 7; + + values[5] = static_cast(*ptr++ & 0x7f) << 30; + values[5] |= static_cast(*ptr++) << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = static_cast(*ptr++ & 3) << 35; + values[6] |= static_cast(*ptr++) << 27; + values[6] |= *ptr++ << 19; + values[6] |= *ptr++ << 11; + values[6] |= *ptr++ << 3; + values[6] |= *ptr >> 5; + + values[7] = static_cast(*ptr++ & 0x1f) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits38(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 30; + values[0] |= *ptr++ << 22; + values[0] |= *ptr++ << 14; + values[0] |= *ptr++ << 6; + values[0] |= *ptr >> 2; + + values[1] = static_cast(*ptr++ & 3) << 36; + values[1] |= static_cast(*ptr++) << 28; + values[1] |= *ptr++ << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = static_cast(*ptr++ & 0xf) << 34; + values[2] |= static_cast(*ptr++) << 26; + values[2] |= *ptr++ << 18; + values[2] |= *ptr++ << 10; + values[2] |= *ptr++ << 2; + values[2] |= *ptr >> 6; + + values[3] = static_cast(*ptr++ & 0x3f) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 30; + values[4] |= *ptr++ << 22; + values[4] |= *ptr++ << 14; + values[4] |= *ptr++ << 6; + values[4] |= *ptr >> 2; + + values[5] = static_cast(*ptr++ & 3) << 36; + values[5] |= static_cast(*ptr++) << 28; + values[5] |= *ptr++ << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = static_cast(*ptr++ & 0xf) << 34; + values[6] |= static_cast(*ptr++) << 26; + values[6] |= *ptr++ << 18; + values[6] |= *ptr++ << 10; + values[6] |= *ptr++ << 2; + values[6] |= *ptr >> 6; + + values[7] = static_cast(*ptr++ & 0x3f) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits39(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 31; + values[0] |= *ptr++ << 23; + values[0] |= *ptr++ << 15; + values[0] |= *ptr++ << 7; + values[0] |= *ptr >> 1; + + values[1] = static_cast(*ptr++ & 1) << 38; + values[1] |= static_cast(*ptr++) << 30; + values[1] |= *ptr++ << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 37; + values[2] |= static_cast(*ptr++) << 29; + values[2] |= *ptr++ << 21; + values[2] |= *ptr++ << 13; + values[2] |= *ptr++ << 5; + values[2] |= *ptr >> 3; + + values[3] = static_cast(*ptr++ & 7) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 35; + values[4] |= static_cast(*ptr++) << 27; + values[4] |= *ptr++ << 19; + values[4] |= *ptr++ << 11; + values[4] |= *ptr++ << 3; + values[4] |= *ptr >> 5; + + values[5] = static_cast(*ptr++ & 0x1f) << 34; + values[5] |= static_cast(*ptr++) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = static_cast(*ptr++ & 0x3f) << 33; + values[6] |= static_cast(*ptr++) << 25; + values[6] |= *ptr++ << 17; + values[6] |= *ptr++ << 9; + values[6] |= *ptr++ << 1; + values[6] |= *ptr >> 7; + + values[7] = static_cast(*ptr++ & 0x7f) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits40(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 32; + values[0] |= static_cast(*ptr++) << 24; + values[0] |= *ptr++ << 16; + values[0] |= *ptr++ << 8; + values[0] |= *ptr++; + values[1] = static_cast(*ptr++) << 32; + values[1] |= static_cast(*ptr++) << 24; + values[1] |= *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + values[2] = static_cast(*ptr++) << 32; + values[2] |= static_cast(*ptr++) << 24; + values[2] |= *ptr++ << 16; + values[2] |= *ptr++ << 8; + values[2] |= *ptr++; + values[3] = static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + values[4] = static_cast(*ptr++) << 32; + values[4] |= static_cast(*ptr++) << 24; + values[4] |= *ptr++ << 16; + values[4] |= *ptr++ << 8; + values[4] |= *ptr++; + values[5] = static_cast(*ptr++) << 32; + values[5] |= static_cast(*ptr++) << 24; + values[5] |= *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + values[6] = static_cast(*ptr++) << 32; + values[6] |= static_cast(*ptr++) << 24; + values[6] |= *ptr++ << 16; + values[6] |= *ptr++ << 8; + values[6] |= *ptr++; + values[7] = static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits41(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 33; + values[0] |= static_cast(*ptr++) << 25; + values[0] |= *ptr++ << 17; + values[0] |= *ptr++ << 9; + values[0] |= *ptr++ << 1; + values[0] |= *ptr >> 7; + + values[1] = static_cast(*ptr++ & 0x7f) << 34; + values[1] |= static_cast(*ptr++) << 26; + values[1] |= *ptr++ << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = static_cast(*ptr++ & 0x3f) << 35; + values[2] |= static_cast(*ptr++) << 27; + values[2] |= *ptr++ << 19; + values[2] |= *ptr++ << 11; + values[2] |= *ptr++ << 3; + values[2] |= *ptr >> 5; + + values[3] = static_cast(*ptr++ & 0x1f) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 37; + values[4] |= static_cast(*ptr++) << 29; + values[4] |= *ptr++ << 21; + values[4] |= *ptr++ << 13; + values[4] |= *ptr++ << 5; + values[4] |= *ptr >> 3; + + values[5] = static_cast(*ptr++ & 7) << 38; + values[5] |= static_cast(*ptr++) << 30; + values[5] |= *ptr++ << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = static_cast(*ptr++ & 3) << 39; + values[6] |= static_cast(*ptr++) << 31; + values[6] |= *ptr++ << 23; + values[6] |= *ptr++ << 15; + values[6] |= *ptr++ << 7; + values[6] |= *ptr >> 1; + + values[7] = static_cast(*ptr++ & 1) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits42(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 34; + values[0] |= static_cast(*ptr++) << 26; + values[0] |= *ptr++ << 18; + values[0] |= *ptr++ << 10; + values[0] |= *ptr++ << 2; + values[0] |= *ptr >> 6; + + values[1] = static_cast(*ptr++ & 0x3f) << 36; + values[1] |= static_cast(*ptr++) << 28; + values[1] |= *ptr++ << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = static_cast(*ptr++ & 0xf) << 38; + values[2] |= static_cast(*ptr++) << 30; + values[2] |= *ptr++ << 22; + values[2] |= *ptr++ << 14; + values[2] |= *ptr++ << 6; + values[2] |= *ptr >> 2; + + values[3] = static_cast(*ptr++ & 3) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 34; + values[4] |= static_cast(*ptr++) << 26; + values[4] |= *ptr++ << 18; + values[4] |= *ptr++ << 10; + values[4] |= *ptr++ << 2; + values[4] |= *ptr >> 6; + + values[5] = static_cast(*ptr++ & 0x3f) << 36; + values[5] |= static_cast(*ptr++) << 28; + values[5] |= *ptr++ << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = static_cast(*ptr++ & 0xf) << 38; + values[6] |= static_cast(*ptr++) << 30; + values[6] |= *ptr++ << 22; + values[6] |= *ptr++ << 14; + values[6] |= *ptr++ << 6; + values[6] |= *ptr >> 2; + + values[7] = static_cast(*ptr++ & 3) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits43(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 35; + values[0] |= static_cast(*ptr++) << 27; + values[0] |= *ptr++ << 19; + values[0] |= *ptr++ << 11; + values[0] |= *ptr++ << 3; + values[0] |= *ptr >> 5; + + values[1] = static_cast(*ptr++ & 0x1f) << 38; + values[1] |= static_cast(*ptr++) << 30; + values[1] |= *ptr++ << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 41; + values[2] |= static_cast(*ptr++) << 33; + values[2] |= static_cast(*ptr++) << 25; + values[2] |= *ptr++ << 17; + values[2] |= *ptr++ << 9; + values[2] |= *ptr++ << 1; + values[2] |= *ptr >> 7; + + values[3] = static_cast(*ptr++ & 0x7f) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 39; + values[4] |= static_cast(*ptr++) << 31; + values[4] |= *ptr++ << 23; + values[4] |= *ptr++ << 15; + values[4] |= *ptr++ << 7; + values[4] |= *ptr >> 1; + + values[5] = static_cast(*ptr++ & 1) << 42; + values[5] |= static_cast(*ptr++) << 34; + values[5] |= static_cast(*ptr++) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = static_cast(*ptr++ & 0x3f) << 37; + values[6] |= static_cast(*ptr++) << 29; + values[6] |= *ptr++ << 21; + values[6] |= *ptr++ << 13; + values[6] |= *ptr++ << 5; + values[6] |= *ptr >> 3; + + values[7] = static_cast(*ptr++ & 7) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits44(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 36; + values[0] |= static_cast(*ptr++) << 28; + values[0] |= *ptr++ << 20; + values[0] |= *ptr++ << 12; + values[0] |= *ptr++ << 4; + values[0] |= *ptr >> 4; + + values[1] = static_cast(*ptr++ & 0xf) << 40; + values[1] |= static_cast(*ptr++) << 32; + values[1] |= static_cast(*ptr++) << 24; + values[1] |= *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + + values[2] = static_cast(*ptr++) << 36; + values[2] |= static_cast(*ptr++) << 28; + values[2] |= *ptr++ << 20; + values[2] |= *ptr++ << 12; + values[2] |= *ptr++ << 4; + values[2] |= *ptr >> 4; + + values[3] = static_cast(*ptr++ & 0xf) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 36; + values[4] |= static_cast(*ptr++) << 28; + values[4] |= *ptr++ << 20; + values[4] |= *ptr++ << 12; + values[4] |= *ptr++ << 4; + values[4] |= *ptr >> 4; + + values[5] = static_cast(*ptr++ & 0xf) << 40; + values[5] |= static_cast(*ptr++) << 32; + values[5] |= static_cast(*ptr++) << 24; + values[5] |= *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + + values[6] = static_cast(*ptr++) << 36; + values[6] |= static_cast(*ptr++) << 28; + values[6] |= *ptr++ << 20; + values[6] |= *ptr++ << 12; + values[6] |= *ptr++ << 4; + values[6] |= *ptr >> 4; + + values[7] = static_cast(*ptr++ & 0xf) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits45(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 37; + values[0] |= static_cast(*ptr++) << 29; + values[0] |= *ptr++ << 21; + values[0] |= *ptr++ << 13; + values[0] |= *ptr++ << 5; + values[0] |= *ptr >> 3; + + values[1] = static_cast(*ptr++ & 7) << 42; + values[1] |= static_cast(*ptr++) << 34; + values[1] |= static_cast(*ptr++) << 26; + values[1] |= *ptr++ << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = static_cast(*ptr++ & 0x3f) << 39; + values[2] |= static_cast(*ptr++) << 31; + values[2] |= static_cast(*ptr++) << 23; + values[2] |= *ptr++ << 15; + values[2] |= *ptr++ << 7; + values[2] |= *ptr >> 1; + + values[3] = static_cast(*ptr++ & 1) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 41; + values[4] |= static_cast(*ptr++) << 33; + values[4] |= static_cast(*ptr++) << 25; + values[4] |= *ptr++ << 17; + values[4] |= *ptr++ << 9; + values[4] |= *ptr++ << 1; + values[4] |= *ptr >> 7; + + values[5] = static_cast(*ptr++ & 0x7f) << 38; + values[5] |= static_cast(*ptr++) << 30; + values[5] |= static_cast(*ptr++) << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = static_cast(*ptr++ & 3) << 43; + values[6] |= static_cast(*ptr++) << 35; + values[6] |= static_cast(*ptr++) << 27; + values[6] |= *ptr++ << 19; + values[6] |= *ptr++ << 11; + values[6] |= *ptr++ << 3; + values[6] |= *ptr >> 5; + + values[7] = static_cast(*ptr++ & 0x1f) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits46(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 38; + values[0] |= static_cast(*ptr++) << 30; + values[0] |= *ptr++ << 22; + values[0] |= *ptr++ << 14; + values[0] |= *ptr++ << 6; + values[0] |= *ptr >> 2; + + values[1] = static_cast(*ptr++ & 3) << 44; + values[1] |= static_cast(*ptr++) << 36; + values[1] |= static_cast(*ptr++) << 28; + values[1] |= *ptr++ << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = static_cast(*ptr++ & 0xf) << 42; + values[2] |= static_cast(*ptr++) << 34; + values[2] |= static_cast(*ptr++) << 26; + values[2] |= *ptr++ << 18; + values[2] |= *ptr++ << 10; + values[2] |= *ptr++ << 2; + values[2] |= *ptr >> 6; + + values[3] = static_cast(*ptr++ & 0x3f) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 38; + values[4] |= static_cast(*ptr++) << 30; + values[4] |= *ptr++ << 22; + values[4] |= *ptr++ << 14; + values[4] |= *ptr++ << 6; + values[4] |= *ptr >> 2; + + values[5] = static_cast(*ptr++ & 3) << 44; + values[5] |= static_cast(*ptr++) << 36; + values[5] |= static_cast(*ptr++) << 28; + values[5] |= *ptr++ << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = static_cast(*ptr++ & 0xf) << 42; + values[6] |= static_cast(*ptr++) << 34; + values[6] |= static_cast(*ptr++) << 26; + values[6] |= *ptr++ << 18; + values[6] |= *ptr++ << 10; + values[6] |= *ptr++ << 2; + values[6] |= *ptr >> 6; + + values[7] = static_cast(*ptr++ & 0x3f) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits47(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 39; + values[0] |= static_cast(*ptr++) << 31; + values[0] |= *ptr++ << 23; + values[0] |= *ptr++ << 15; + values[0] |= *ptr++ << 7; + values[0] |= *ptr >> 1; + + values[1] = static_cast(*ptr++ & 1) << 46; + values[1] |= static_cast(*ptr++) << 38; + values[1] |= static_cast(*ptr++) << 30; + values[1] |= *ptr++ << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 45; + values[2] |= static_cast(*ptr++) << 37; + values[2] |= static_cast(*ptr++) << 29; + values[2] |= *ptr++ << 21; + values[2] |= *ptr++ << 13; + values[2] |= *ptr++ << 5; + values[2] |= *ptr >> 3; + + values[3] = static_cast(*ptr++ & 7) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 43; + values[4] |= static_cast(*ptr++) << 35; + values[4] |= static_cast(*ptr++) << 27; + values[4] |= *ptr++ << 19; + values[4] |= *ptr++ << 11; + values[4] |= *ptr++ << 3; + values[4] |= *ptr >> 5; + + values[5] = static_cast(*ptr++ & 0x1f) << 42; + values[5] |= static_cast(*ptr++) << 34; + values[5] |= static_cast(*ptr++) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = static_cast(*ptr++ & 0x3f) << 41; + values[6] |= static_cast(*ptr++) << 33; + values[6] |= static_cast(*ptr++) << 25; + values[6] |= *ptr++ << 17; + values[6] |= *ptr++ << 9; + values[6] |= *ptr++ << 1; + values[6] |= *ptr >> 7; + + values[7] = static_cast(*ptr++ & 0x7f) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits48(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 40; + values[0] |= static_cast(*ptr++) << 32; + values[0] |= static_cast(*ptr++) << 24; + values[0] |= *ptr++ << 16; + values[0] |= *ptr++ << 8; + values[0] |= *ptr++; + values[1] = static_cast(*ptr++) << 40; + values[1] |= static_cast(*ptr++) << 32; + values[1] |= static_cast(*ptr++) << 24; + values[1] |= *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + values[2] = static_cast(*ptr++) << 40; + values[2] |= static_cast(*ptr++) << 32; + values[2] |= static_cast(*ptr++) << 24; + values[2] |= *ptr++ << 16; + values[2] |= *ptr++ << 8; + values[2] |= *ptr++; + values[3] = static_cast(*ptr++) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + values[4] = static_cast(*ptr++) << 40; + values[4] |= static_cast(*ptr++) << 32; + values[4] |= static_cast(*ptr++) << 24; + values[4] |= *ptr++ << 16; + values[4] |= *ptr++ << 8; + values[4] |= *ptr++; + values[5] = static_cast(*ptr++) << 40; + values[5] |= static_cast(*ptr++) << 32; + values[5] |= static_cast(*ptr++) << 24; + values[5] |= *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + values[6] = static_cast(*ptr++) << 40; + values[6] |= static_cast(*ptr++) << 32; + values[6] |= static_cast(*ptr++) << 24; + values[6] |= *ptr++ << 16; + values[6] |= *ptr++ << 8; + values[6] |= *ptr++; + values[7] = static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits49(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 41; + values[0] |= static_cast(*ptr++) << 33; + values[0] |= static_cast(*ptr++) << 25; + values[0] |= *ptr++ << 17; + values[0] |= *ptr++ << 9; + values[0] |= *ptr++ << 1; + values[0] |= *ptr >> 7; + + values[1] = static_cast(*ptr++ & 0x7f) << 42; + values[1] |= static_cast(*ptr++) << 34; + values[1] |= static_cast(*ptr++) << 26; + values[1] |= *ptr++ << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = static_cast(*ptr++ & 0x3f) << 43; + values[2] |= static_cast(*ptr++) << 35; + values[2] |= static_cast(*ptr++) << 27; + values[2] |= *ptr++ << 19; + values[2] |= *ptr++ << 11; + values[2] |= *ptr++ << 3; + values[2] |= *ptr >> 5; + + values[3] = static_cast(*ptr++ & 0x1f) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 45; + values[4] |= static_cast(*ptr++) << 37; + values[4] |= static_cast(*ptr++) << 29; + values[4] |= *ptr++ << 21; + values[4] |= *ptr++ << 13; + values[4] |= *ptr++ << 5; + values[4] |= *ptr >> 3; + + values[5] = static_cast(*ptr++ & 7) << 46; + values[5] |= static_cast(*ptr++) << 38; + values[5] |= static_cast(*ptr++) << 30; + values[5] |= *ptr++ << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = static_cast(*ptr++ & 3) << 47; + values[6] |= static_cast(*ptr++) << 39; + values[6] |= static_cast(*ptr++) << 31; + values[6] |= *ptr++ << 23; + values[6] |= *ptr++ << 15; + values[6] |= *ptr++ << 7; + values[6] |= *ptr >> 1; + + values[7] = static_cast(*ptr++ & 1) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits50(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 42; + values[0] |= static_cast(*ptr++) << 34; + values[0] |= static_cast(*ptr++) << 26; + values[0] |= *ptr++ << 18; + values[0] |= *ptr++ << 10; + values[0] |= *ptr++ << 2; + values[0] |= *ptr >> 6; + + values[1] = static_cast(*ptr++ & 0x3f) << 44; + values[1] |= static_cast(*ptr++) << 36; + values[1] |= static_cast(*ptr++) << 28; + values[1] |= *ptr++ << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = static_cast(*ptr++ & 0xf) << 46; + values[2] |= static_cast(*ptr++) << 38; + values[2] |= static_cast(*ptr++) << 30; + values[2] |= *ptr++ << 22; + values[2] |= *ptr++ << 14; + values[2] |= *ptr++ << 6; + values[2] |= *ptr >> 2; + + values[3] = static_cast(*ptr++ & 3) << 48; + values[3] |= static_cast(*ptr++) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 42; + values[4] |= static_cast(*ptr++) << 34; + values[4] |= static_cast(*ptr++) << 26; + values[4] |= *ptr++ << 18; + values[4] |= *ptr++ << 10; + values[4] |= *ptr++ << 2; + values[4] |= *ptr >> 6; + + values[5] = static_cast(*ptr++ & 0x3f) << 44; + values[5] |= static_cast(*ptr++) << 36; + values[5] |= static_cast(*ptr++) << 28; + values[5] |= *ptr++ << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = static_cast(*ptr++ & 0xf) << 46; + values[6] |= static_cast(*ptr++) << 38; + values[6] |= static_cast(*ptr++) << 30; + values[6] |= *ptr++ << 22; + values[6] |= *ptr++ << 14; + values[6] |= *ptr++ << 6; + values[6] |= *ptr >> 2; + + values[7] = static_cast(*ptr++ & 3) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits51(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 43; + values[0] |= static_cast(*ptr++) << 35; + values[0] |= static_cast(*ptr++) << 27; + values[0] |= *ptr++ << 19; + values[0] |= *ptr++ << 11; + values[0] |= *ptr++ << 3; + values[0] |= *ptr >> 5; + + values[1] = static_cast(*ptr++ & 0x1f) << 46; + values[1] |= static_cast(*ptr++) << 38; + values[1] |= static_cast(*ptr++) << 30; + values[1] |= *ptr++ << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 49; + values[2] |= static_cast(*ptr++) << 41; + values[2] |= static_cast(*ptr++) << 33; + values[2] |= static_cast(*ptr++) << 25; + values[2] |= *ptr++ << 17; + values[2] |= *ptr++ << 9; + values[2] |= *ptr++ << 1; + values[2] |= *ptr >> 7; + + values[3] = static_cast(*ptr++ & 0x7f) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 47; + values[4] |= static_cast(*ptr++) << 39; + values[4] |= static_cast(*ptr++) << 31; + values[4] |= *ptr++ << 23; + values[4] |= *ptr++ << 15; + values[4] |= *ptr++ << 7; + values[4] |= *ptr >> 1; + + values[5] = static_cast(*ptr++ & 1) << 50; + values[5] |= static_cast(*ptr++) << 42; + values[5] |= static_cast(*ptr++) << 34; + values[5] |= static_cast(*ptr++) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = static_cast(*ptr++ & 0x3f) << 45; + values[6] |= static_cast(*ptr++) << 37; + values[6] |= static_cast(*ptr++) << 29; + values[6] |= *ptr++ << 21; + values[6] |= *ptr++ << 13; + values[6] |= *ptr++ << 5; + values[6] |= *ptr >> 3; + + values[7] = static_cast(*ptr++ & 7) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits52(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 44; + values[0] |= static_cast(*ptr++) << 36; + values[0] |= static_cast(*ptr++) << 28; + values[0] |= *ptr++ << 20; + values[0] |= *ptr++ << 12; + values[0] |= *ptr++ << 4; + values[0] |= *ptr >> 4; + + values[1] = static_cast(*ptr++ & 0xf) << 48; + values[1] |= static_cast(*ptr++) << 40; + values[1] |= static_cast(*ptr++) << 32; + values[1] |= static_cast(*ptr++) << 24; + values[1] |= *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + + values[2] = static_cast(*ptr++) << 44; + values[2] |= static_cast(*ptr++) << 36; + values[2] |= static_cast(*ptr++) << 28; + values[2] |= *ptr++ << 20; + values[2] |= *ptr++ << 12; + values[2] |= *ptr++ << 4; + values[2] |= *ptr >> 4; + + values[3] = static_cast(*ptr++ & 0xf) << 48; + values[3] |= static_cast(*ptr++) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 44; + values[4] |= static_cast(*ptr++) << 36; + values[4] |= static_cast(*ptr++) << 28; + values[4] |= *ptr++ << 20; + values[4] |= *ptr++ << 12; + values[4] |= *ptr++ << 4; + values[4] |= *ptr >> 4; + + values[5] = static_cast(*ptr++ & 0xf) << 48; + values[5] |= static_cast(*ptr++) << 40; + values[5] |= static_cast(*ptr++) << 32; + values[5] |= static_cast(*ptr++) << 24; + values[5] |= *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + + values[6] = static_cast(*ptr++) << 44; + values[6] |= static_cast(*ptr++) << 36; + values[6] |= static_cast(*ptr++) << 28; + values[6] |= *ptr++ << 20; + values[6] |= *ptr++ << 12; + values[6] |= *ptr++ << 4; + values[6] |= *ptr >> 4; + + values[7] = static_cast(*ptr++ & 0xf) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits53(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 45; + values[0] |= static_cast(*ptr++) << 37; + values[0] |= static_cast(*ptr++) << 29; + values[0] |= *ptr++ << 21; + values[0] |= *ptr++ << 13; + values[0] |= *ptr++ << 5; + values[0] |= *ptr >> 3; + + values[1] = static_cast(*ptr++ & 7) << 50; + values[1] |= static_cast(*ptr++) << 42; + values[1] |= static_cast(*ptr++) << 34; + values[1] |= static_cast(*ptr++) << 26; + values[1] |= *ptr++ << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = static_cast(*ptr++ & 0x3f) << 47; + values[2] |= static_cast(*ptr++) << 39; + values[2] |= static_cast(*ptr++) << 31; + values[2] |= static_cast(*ptr++) << 23; + values[2] |= *ptr++ << 15; + values[2] |= *ptr++ << 7; + values[2] |= *ptr >> 1; + + values[3] = static_cast(*ptr++ & 1) << 52; + values[3] |= static_cast(*ptr++) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 49; + values[4] |= static_cast(*ptr++) << 41; + values[4] |= static_cast(*ptr++) << 33; + values[4] |= static_cast(*ptr++) << 25; + values[4] |= *ptr++ << 17; + values[4] |= *ptr++ << 9; + values[4] |= *ptr++ << 1; + values[4] |= *ptr >> 7; + + values[5] = static_cast(*ptr++ & 0x7f) << 46; + values[5] |= static_cast(*ptr++) << 38; + values[5] |= static_cast(*ptr++) << 30; + values[5] |= *ptr++ << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = static_cast(*ptr++ & 3) << 51; + values[6] |= static_cast(*ptr++) << 43; + values[6] |= static_cast(*ptr++) << 35; + values[6] |= static_cast(*ptr++) << 27; + values[6] |= *ptr++ << 19; + values[6] |= *ptr++ << 11; + values[6] |= *ptr++ << 3; + values[6] |= *ptr >> 5; + + values[7] = static_cast(*ptr++ & 0x1f) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits54(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 46; + values[0] |= static_cast(*ptr++) << 38; + values[0] |= static_cast(*ptr++) << 30; + values[0] |= *ptr++ << 22; + values[0] |= *ptr++ << 14; + values[0] |= *ptr++ << 6; + values[0] |= *ptr >> 2; + + values[1] = static_cast(*ptr++ & 3) << 52; + values[1] |= static_cast(*ptr++) << 44; + values[1] |= static_cast(*ptr++) << 36; + values[1] |= static_cast(*ptr++) << 28; + values[1] |= *ptr++ << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = static_cast(*ptr++ & 0xf) << 50; + values[2] |= static_cast(*ptr++) << 42; + values[2] |= static_cast(*ptr++) << 34; + values[2] |= static_cast(*ptr++) << 26; + values[2] |= *ptr++ << 18; + values[2] |= *ptr++ << 10; + values[2] |= *ptr++ << 2; + values[2] |= *ptr >> 6; + + values[3] = static_cast(*ptr++ & 0x3f) << 48; + values[3] |= static_cast(*ptr++) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 46; + values[4] |= static_cast(*ptr++) << 38; + values[4] |= static_cast(*ptr++) << 30; + values[4] |= *ptr++ << 22; + values[4] |= *ptr++ << 14; + values[4] |= *ptr++ << 6; + values[4] |= *ptr >> 2; + + values[5] = static_cast(*ptr++ & 3) << 52; + values[5] |= static_cast(*ptr++) << 44; + values[5] |= static_cast(*ptr++) << 36; + values[5] |= static_cast(*ptr++) << 28; + values[5] |= *ptr++ << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = static_cast(*ptr++ & 0xf) << 50; + values[6] |= static_cast(*ptr++) << 42; + values[6] |= static_cast(*ptr++) << 34; + values[6] |= static_cast(*ptr++) << 26; + values[6] |= *ptr++ << 18; + values[6] |= *ptr++ << 10; + values[6] |= *ptr++ << 2; + values[6] |= *ptr >> 6; + + values[7] = static_cast(*ptr++ & 0x3f) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr++; +} + +static inline void unpackBits55(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 47; + values[0] |= static_cast(*ptr++) << 39; + values[0] |= static_cast(*ptr++) << 31; + values[0] |= *ptr++ << 23; + values[0] |= *ptr++ << 15; + values[0] |= *ptr++ << 7; + values[0] |= *ptr >> 1; + + values[1] = static_cast(*ptr++ & 1) << 54; + values[1] |= static_cast(*ptr++) << 46; + values[1] |= static_cast(*ptr++) << 38; + values[1] |= static_cast(*ptr++) << 30; + values[1] |= *ptr++ << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 53; + values[2] |= static_cast(*ptr++) << 45; + values[2] |= static_cast(*ptr++) << 37; + values[2] |= static_cast(*ptr++) << 29; + values[2] |= *ptr++ << 21; + values[2] |= *ptr++ << 13; + values[2] |= *ptr++ << 5; + values[2] |= *ptr >> 3; + + values[3] = static_cast(*ptr++ & 7) << 52; + values[3] |= static_cast(*ptr++) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 51; + values[4] |= static_cast(*ptr++) << 43; + values[4] |= static_cast(*ptr++) << 35; + values[4] |= static_cast(*ptr++) << 27; + values[4] |= *ptr++ << 19; + values[4] |= *ptr++ << 11; + values[4] |= *ptr++ << 3; + values[4] |= *ptr >> 5; + + values[5] = static_cast(*ptr++ & 0x1f) << 50; + values[5] |= static_cast(*ptr++) << 42; + values[5] |= static_cast(*ptr++) << 34; + values[5] |= static_cast(*ptr++) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = static_cast(*ptr++ & 0x3f) << 49; + values[6] |= static_cast(*ptr++) << 41; + values[6] |= static_cast(*ptr++) << 33; + values[6] |= static_cast(*ptr++) << 25; + values[6] |= *ptr++ << 17; + values[6] |= *ptr++ << 9; + values[6] |= *ptr++ << 1; + values[6] |= *ptr >> 7; + + values[7] = static_cast(*ptr++ & 0x7f) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits56(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 48; + values[0] |= static_cast(*ptr++) << 40; + values[0] |= static_cast(*ptr++) << 32; + values[0] |= static_cast(*ptr++) << 24; + values[0] |= *ptr++ << 16; + values[0] |= *ptr++ << 8; + values[0] |= *ptr++; + values[1] = static_cast(*ptr++) << 48; + values[1] |= static_cast(*ptr++) << 40; + values[1] |= static_cast(*ptr++) << 32; + values[1] |= static_cast(*ptr++) << 24; + values[1] |= *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + values[2] = static_cast(*ptr++) << 48; + values[2] |= static_cast(*ptr++) << 40; + values[2] |= static_cast(*ptr++) << 32; + values[2] |= static_cast(*ptr++) << 24; + values[2] |= *ptr++ << 16; + values[2] |= *ptr++ << 8; + values[2] |= *ptr++; + values[3] = static_cast(*ptr++) << 48; + values[3] |= static_cast(*ptr++) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + values[4] = static_cast(*ptr++) << 48; + values[4] |= static_cast(*ptr++) << 40; + values[4] |= static_cast(*ptr++) << 32; + values[4] |= static_cast(*ptr++) << 24; + values[4] |= *ptr++ << 16; + values[4] |= *ptr++ << 8; + values[4] |= *ptr++; + values[5] = static_cast(*ptr++) << 48; + values[5] |= static_cast(*ptr++) << 40; + values[5] |= static_cast(*ptr++) << 32; + values[5] |= static_cast(*ptr++) << 24; + values[5] |= *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + values[6] = static_cast(*ptr++) << 48; + values[6] |= static_cast(*ptr++) << 40; + values[6] |= static_cast(*ptr++) << 32; + values[6] |= static_cast(*ptr++) << 24; + values[6] |= *ptr++ << 16; + values[6] |= *ptr++ << 8; + values[6] |= *ptr++; + values[7] = static_cast(*ptr++) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits57(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 49; + values[0] |= static_cast(*ptr++) << 41; + values[0] |= static_cast(*ptr++) << 33; + values[0] |= static_cast(*ptr++) << 25; + values[0] |= *ptr++ << 17; + values[0] |= *ptr++ << 9; + values[0] |= *ptr++ << 1; + values[0] |= *ptr >> 7; + + values[1] = static_cast(*ptr++ & 0x7f) << 50; + values[1] |= static_cast(*ptr++) << 42; + values[1] |= static_cast(*ptr++) << 34; + values[1] |= static_cast(*ptr++) << 26; + values[1] |= *ptr++ << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = static_cast(*ptr++ & 0x3f) << 51; + values[2] |= static_cast(*ptr++) << 43; + values[2] |= static_cast(*ptr++) << 35; + values[2] |= static_cast(*ptr++) << 27; + values[2] |= *ptr++ << 19; + values[2] |= *ptr++ << 11; + values[2] |= *ptr++ << 3; + values[2] |= *ptr >> 5; + + values[3] = static_cast(*ptr++ & 0x1f) << 52; + values[3] |= static_cast(*ptr++) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 53; + values[4] |= static_cast(*ptr++) << 45; + values[4] |= static_cast(*ptr++) << 37; + values[4] |= static_cast(*ptr++) << 29; + values[4] |= *ptr++ << 21; + values[4] |= *ptr++ << 13; + values[4] |= *ptr++ << 5; + values[4] |= *ptr >> 3; + + values[5] = static_cast(*ptr++ & 7) << 54; + values[5] |= static_cast(*ptr++) << 46; + values[5] |= static_cast(*ptr++) << 38; + values[5] |= static_cast(*ptr++) << 30; + values[5] |= *ptr++ << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = static_cast(*ptr++ & 3) << 55; + values[6] |= static_cast(*ptr++) << 47; + values[6] |= static_cast(*ptr++) << 39; + values[6] |= static_cast(*ptr++) << 31; + values[6] |= *ptr++ << 23; + values[6] |= *ptr++ << 15; + values[6] |= *ptr++ << 7; + values[6] |= *ptr >> 1; + + values[7] = static_cast(*ptr++ & 1) << 56; + values[7] |= static_cast(*ptr++) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits58(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 50; + values[0] |= static_cast(*ptr++) << 42; + values[0] |= static_cast(*ptr++) << 34; + values[0] |= static_cast(*ptr++) << 26; + values[0] |= *ptr++ << 18; + values[0] |= *ptr++ << 10; + values[0] |= *ptr++ << 2; + values[0] |= *ptr >> 6; + + values[1] = static_cast(*ptr++ & 0x3f) << 52; + values[1] |= static_cast(*ptr++) << 44; + values[1] |= static_cast(*ptr++) << 36; + values[1] |= static_cast(*ptr++) << 28; + values[1] |= *ptr++ << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = static_cast(*ptr++ & 0xf) << 54; + values[2] |= static_cast(*ptr++) << 46; + values[2] |= static_cast(*ptr++) << 38; + values[2] |= static_cast(*ptr++) << 30; + values[2] |= *ptr++ << 22; + values[2] |= *ptr++ << 14; + values[2] |= *ptr++ << 6; + values[2] |= *ptr >> 2; + + values[3] = static_cast(*ptr++ & 3) << 56; + values[3] |= static_cast(*ptr++) << 48; + values[3] |= static_cast(*ptr++) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 50; + values[4] |= static_cast(*ptr++) << 42; + values[4] |= static_cast(*ptr++) << 34; + values[4] |= static_cast(*ptr++) << 26; + values[4] |= *ptr++ << 18; + values[4] |= *ptr++ << 10; + values[4] |= *ptr++ << 2; + values[4] |= *ptr >> 6; + + values[5] = static_cast(*ptr++ & 0x3f) << 52; + values[5] |= static_cast(*ptr++) << 44; + values[5] |= static_cast(*ptr++) << 36; + values[5] |= static_cast(*ptr++) << 28; + values[5] |= *ptr++ << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = static_cast(*ptr++ & 0xf) << 54; + values[6] |= static_cast(*ptr++) << 46; + values[6] |= static_cast(*ptr++) << 38; + values[6] |= static_cast(*ptr++) << 30; + values[6] |= *ptr++ << 22; + values[6] |= *ptr++ << 14; + values[6] |= *ptr++ << 6; + values[6] |= *ptr >> 2; + + values[7] = static_cast(*ptr++ & 3) << 56; + values[7] |= static_cast(*ptr++) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr++; +} + +static inline void unpackBits59(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 51; + values[0] |= static_cast(*ptr++) << 43; + values[0] |= static_cast(*ptr++) << 35; + values[0] |= static_cast(*ptr++) << 27; + values[0] |= *ptr++ << 19; + values[0] |= *ptr++ << 11; + values[0] |= *ptr++ << 3; + values[0] |= *ptr >> 5; + + values[1] = static_cast(*ptr++ & 0x1f) << 54; + values[1] |= static_cast(*ptr++) << 46; + values[1] |= static_cast(*ptr++) << 38; + values[1] |= static_cast(*ptr++) << 30; + values[1] |= *ptr++ << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 57; + values[2] |= static_cast(*ptr++) << 49; + values[2] |= static_cast(*ptr++) << 41; + values[2] |= static_cast(*ptr++) << 33; + values[2] |= static_cast(*ptr++) << 25; + values[2] |= *ptr++ << 17; + values[2] |= *ptr++ << 9; + values[2] |= *ptr++ << 1; + values[2] |= *ptr >> 7; + + values[3] = static_cast(*ptr++ & 0x7f) << 52; + values[3] |= static_cast(*ptr++) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 55; + values[4] |= static_cast(*ptr++) << 47; + values[4] |= static_cast(*ptr++) << 39; + values[4] |= static_cast(*ptr++) << 31; + values[4] |= *ptr++ << 23; + values[4] |= *ptr++ << 15; + values[4] |= *ptr++ << 7; + values[4] |= *ptr >> 1; + + values[5] = static_cast(*ptr++ & 1) << 58; + values[5] |= static_cast(*ptr++) << 50; + values[5] |= static_cast(*ptr++) << 42; + values[5] |= static_cast(*ptr++) << 34; + values[5] |= static_cast(*ptr++) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = static_cast(*ptr++ & 0x3f) << 53; + values[6] |= static_cast(*ptr++) << 45; + values[6] |= static_cast(*ptr++) << 37; + values[6] |= static_cast(*ptr++) << 29; + values[6] |= *ptr++ << 21; + values[6] |= *ptr++ << 13; + values[6] |= *ptr++ << 5; + values[6] |= *ptr >> 3; + + values[7] = static_cast(*ptr++ & 7) << 56; + values[7] |= static_cast(*ptr++) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits60(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 52; + values[0] |= static_cast(*ptr++) << 44; + values[0] |= static_cast(*ptr++) << 36; + values[0] |= static_cast(*ptr++) << 28; + values[0] |= *ptr++ << 20; + values[0] |= *ptr++ << 12; + values[0] |= *ptr++ << 4; + values[0] |= *ptr >> 4; + + values[1] = static_cast(*ptr++ & 0xf) << 56; + values[1] |= static_cast(*ptr++) << 48; + values[1] |= static_cast(*ptr++) << 40; + values[1] |= static_cast(*ptr++) << 32; + values[1] |= static_cast(*ptr++) << 24; + values[1] |= *ptr++ << 16; + values[1] |= *ptr++ << 8; + values[1] |= *ptr++; + + values[2] = static_cast(*ptr++) << 52; + values[2] |= static_cast(*ptr++) << 44; + values[2] |= static_cast(*ptr++) << 36; + values[2] |= static_cast(*ptr++) << 28; + values[2] |= *ptr++ << 20; + values[2] |= *ptr++ << 12; + values[2] |= *ptr++ << 4; + values[2] |= *ptr >> 4; + + values[3] = static_cast(*ptr++ & 0xf) << 56; + values[3] |= static_cast(*ptr++) << 48; + values[3] |= static_cast(*ptr++) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 52; + values[4] |= static_cast(*ptr++) << 44; + values[4] |= static_cast(*ptr++) << 36; + values[4] |= static_cast(*ptr++) << 28; + values[4] |= *ptr++ << 20; + values[4] |= *ptr++ << 12; + values[4] |= *ptr++ << 4; + values[4] |= *ptr >> 4; + + values[5] = static_cast(*ptr++ & 0xf) << 56; + values[5] |= static_cast(*ptr++) << 48; + values[5] |= static_cast(*ptr++) << 40; + values[5] |= static_cast(*ptr++) << 32; + values[5] |= static_cast(*ptr++) << 24; + values[5] |= *ptr++ << 16; + values[5] |= *ptr++ << 8; + values[5] |= *ptr++; + + values[6] = static_cast(*ptr++) << 52; + values[6] |= static_cast(*ptr++) << 44; + values[6] |= static_cast(*ptr++) << 36; + values[6] |= static_cast(*ptr++) << 28; + values[6] |= *ptr++ << 20; + values[6] |= *ptr++ << 12; + values[6] |= *ptr++ << 4; + values[6] |= *ptr >> 4; + + values[7] = static_cast(*ptr++ & 0xf) << 56; + values[7] |= static_cast(*ptr++) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits61(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 53; + values[0] |= static_cast(*ptr++) << 45; + values[0] |= static_cast(*ptr++) << 37; + values[0] |= static_cast(*ptr++) << 29; + values[0] |= *ptr++ << 21; + values[0] |= *ptr++ << 13; + values[0] |= *ptr++ << 5; + values[0] |= *ptr >> 3; + + values[1] = static_cast(*ptr++ & 7) << 58; + values[1] |= static_cast(*ptr++) << 50; + values[1] |= static_cast(*ptr++) << 42; + values[1] |= static_cast(*ptr++) << 34; + values[1] |= static_cast(*ptr++) << 26; + values[1] |= *ptr++ << 18; + values[1] |= *ptr++ << 10; + values[1] |= *ptr++ << 2; + values[1] |= *ptr >> 6; + + values[2] = static_cast(*ptr++ & 0x3f) << 55; + values[2] |= static_cast(*ptr++) << 47; + values[2] |= static_cast(*ptr++) << 39; + values[2] |= static_cast(*ptr++) << 31; + values[2] |= *ptr++ << 23; + values[2] |= *ptr++ << 15; + values[2] |= *ptr++ << 7; + values[2] |= *ptr >> 1; + + values[3] = static_cast(*ptr++ & 1) << 60; + values[3] |= static_cast(*ptr++) << 52; + values[3] |= static_cast(*ptr++) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 57; + values[4] |= static_cast(*ptr++) << 49; + values[4] |= static_cast(*ptr++) << 41; + values[4] |= static_cast(*ptr++) << 33; + values[4] |= static_cast(*ptr++) << 25; + values[4] |= *ptr++ << 17; + values[4] |= *ptr++ << 9; + values[4] |= *ptr++ << 1; + values[4] |= *ptr >> 7; + + values[5] = static_cast(*ptr++ & 0x7f) << 54; + values[5] |= static_cast(*ptr++) << 46; + values[5] |= static_cast(*ptr++) << 38; + values[5] |= static_cast(*ptr++) << 30; + values[5] |= *ptr++ << 22; + values[5] |= *ptr++ << 14; + values[5] |= *ptr++ << 6; + values[5] |= *ptr >> 2; + + values[6] = static_cast(*ptr++ & 3) << 59; + values[6] |= static_cast(*ptr++) << 51; + values[6] |= static_cast(*ptr++) << 43; + values[6] |= static_cast(*ptr++) << 35; + values[6] |= static_cast(*ptr++) << 27; + values[6] |= *ptr++ << 19; + values[6] |= *ptr++ << 11; + values[6] |= *ptr++ << 3; + values[6] |= *ptr >> 5; + + values[7] = static_cast(*ptr++ & 0x1f) << 56; + values[7] |= static_cast(*ptr++) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits62(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 54; + values[0] |= static_cast(*ptr++) << 46; + values[0] |= static_cast(*ptr++) << 38; + values[0] |= static_cast(*ptr++) << 30; + values[0] |= *ptr++ << 22; + values[0] |= *ptr++ << 14; + values[0] |= *ptr++ << 6; + values[0] |= *ptr >> 2; + + values[1] = static_cast(*ptr++ & 3) << 60; + values[1] |= static_cast(*ptr++) << 52; + values[1] |= static_cast(*ptr++) << 44; + values[1] |= static_cast(*ptr++) << 36; + values[1] |= static_cast(*ptr++) << 28; + values[1] |= *ptr++ << 20; + values[1] |= *ptr++ << 12; + values[1] |= *ptr++ << 4; + values[1] |= *ptr >> 4; + + values[2] = static_cast(*ptr++ & 0xf) << 58; + values[2] |= static_cast(*ptr++) << 50; + values[2] |= static_cast(*ptr++) << 42; + values[2] |= static_cast(*ptr++) << 34; + values[2] |= static_cast(*ptr++) << 26; + values[2] |= *ptr++ << 18; + values[2] |= *ptr++ << 10; + values[2] |= *ptr++ << 2; + values[2] |= *ptr >> 6; + + values[3] = static_cast(*ptr++ & 0x3f) << 56; + values[3] |= static_cast(*ptr++) << 48; + values[3] |= static_cast(*ptr++) << 40; + values[3] |= static_cast(*ptr++) << 32; + values[3] |= static_cast(*ptr++) << 24; + values[3] |= *ptr++ << 16; + values[3] |= *ptr++ << 8; + values[3] |= *ptr++; + + values[4] = static_cast(*ptr++) << 54; + values[4] |= static_cast(*ptr++) << 46; + values[4] |= static_cast(*ptr++) << 38; + values[4] |= static_cast(*ptr++) << 30; + values[4] |= *ptr++ << 22; + values[4] |= *ptr++ << 14; + values[4] |= *ptr++ << 6; + values[4] |= *ptr >> 2; + + values[5] = static_cast(*ptr++ & 3) << 60; + values[5] |= static_cast(*ptr++) << 52; + values[5] |= static_cast(*ptr++) << 44; + values[5] |= static_cast(*ptr++) << 36; + values[5] |= static_cast(*ptr++) << 28; + values[5] |= *ptr++ << 20; + values[5] |= *ptr++ << 12; + values[5] |= *ptr++ << 4; + values[5] |= *ptr >> 4; + + values[6] = static_cast(*ptr++ & 0xf) << 58; + values[6] |= static_cast(*ptr++) << 50; + values[6] |= static_cast(*ptr++) << 42; + values[6] |= static_cast(*ptr++) << 34; + values[6] |= static_cast(*ptr++) << 26; + values[6] |= *ptr++ << 18; + values[6] |= *ptr++ << 10; + values[6] |= *ptr++ << 2; + values[6] |= *ptr >> 6; + + values[7] = static_cast(*ptr++ & 0x3f) << 56; + values[7] |= static_cast(*ptr++) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void unpackBits63(uint64_t* values, const uint8_t* ptr) { + values[0] = static_cast(*ptr++) << 55; + values[0] |= static_cast(*ptr++) << 47; + values[0] |= static_cast(*ptr++) << 39; + values[0] |= static_cast(*ptr++) << 31; + values[0] |= *ptr++ << 23; + values[0] |= *ptr++ << 15; + values[0] |= *ptr++ << 7; + values[0] |= *ptr >> 1; + + values[1] = static_cast(*ptr++ & 1) << 62; + values[1] |= static_cast(*ptr++) << 54; + values[1] |= static_cast(*ptr++) << 46; + values[1] |= static_cast(*ptr++) << 38; + values[1] |= static_cast(*ptr++) << 30; + values[1] |= *ptr++ << 22; + values[1] |= *ptr++ << 14; + values[1] |= *ptr++ << 6; + values[1] |= *ptr >> 2; + + values[2] = static_cast(*ptr++ & 3) << 61; + values[2] |= static_cast(*ptr++) << 53; + values[2] |= static_cast(*ptr++) << 45; + values[2] |= static_cast(*ptr++) << 37; + values[2] |= static_cast(*ptr++) << 29; + values[2] |= *ptr++ << 21; + values[2] |= *ptr++ << 13; + values[2] |= *ptr++ << 5; + values[2] |= *ptr >> 3; + + values[3] = static_cast(*ptr++ & 7) << 60; + values[3] |= static_cast(*ptr++) << 52; + values[3] |= static_cast(*ptr++) << 44; + values[3] |= static_cast(*ptr++) << 36; + values[3] |= static_cast(*ptr++) << 28; + values[3] |= *ptr++ << 20; + values[3] |= *ptr++ << 12; + values[3] |= *ptr++ << 4; + values[3] |= *ptr >> 4; + + values[4] = static_cast(*ptr++ & 0xf) << 59; + values[4] |= static_cast(*ptr++) << 51; + values[4] |= static_cast(*ptr++) << 43; + values[4] |= static_cast(*ptr++) << 35; + values[4] |= static_cast(*ptr++) << 27; + values[4] |= *ptr++ << 19; + values[4] |= *ptr++ << 11; + values[4] |= *ptr++ << 3; + values[4] |= *ptr >> 5; + + values[5] = static_cast(*ptr++ & 0x1f) << 58; + values[5] |= static_cast(*ptr++) << 50; + values[5] |= static_cast(*ptr++) << 42; + values[5] |= static_cast(*ptr++) << 34; + values[5] |= static_cast(*ptr++) << 26; + values[5] |= *ptr++ << 18; + values[5] |= *ptr++ << 10; + values[5] |= *ptr++ << 2; + values[5] |= *ptr >> 6; + + values[6] = static_cast(*ptr++ & 0x3f) << 57; + values[6] |= static_cast(*ptr++) << 49; + values[6] |= static_cast(*ptr++) << 41; + values[6] |= static_cast(*ptr++) << 33; + values[6] |= static_cast(*ptr++) << 25; + values[6] |= *ptr++ << 17; + values[6] |= *ptr++ << 9; + values[6] |= *ptr++ << 1; + values[6] |= *ptr >> 7; + + values[7] = static_cast(*ptr++ & 0x7f) << 56; + values[7] |= static_cast(*ptr++) << 48; + values[7] |= static_cast(*ptr++) << 40; + values[7] |= static_cast(*ptr++) << 32; + values[7] |= static_cast(*ptr++) << 24; + values[7] |= *ptr++ << 16; + values[7] |= *ptr++ << 8; + values[7] |= *ptr; +} + +static inline void +packBitsBlock8(const uint64_t* values, uint8_t* ptr, uint8_t bits) { + switch (bits) { + case 1: + packBits1(values, ptr); + break; + case 2: + packBits2(values, ptr); + break; + case 3: + packBits3(values, ptr); + break; + case 4: + packBits4(values, ptr); + break; + case 5: + packBits5(values, ptr); + break; + case 6: + packBits6(values, ptr); + break; + case 7: + packBits7(values, ptr); + break; + case 8: + packBits8(values, ptr); + break; + case 9: + packBits9(values, ptr); + break; + case 10: + packBits10(values, ptr); + break; + case 11: + packBits11(values, ptr); + break; + case 12: + packBits12(values, ptr); + break; + case 13: + packBits13(values, ptr); + break; + case 14: + packBits14(values, ptr); + break; + case 15: + packBits15(values, ptr); + break; + case 16: + packBits16(values, ptr); + break; + case 17: + packBits17(values, ptr); + break; + case 18: + packBits18(values, ptr); + break; + case 19: + packBits19(values, ptr); + break; + case 20: + packBits20(values, ptr); + break; + case 21: + packBits21(values, ptr); + break; + case 22: + packBits22(values, ptr); + break; + case 23: + packBits23(values, ptr); + break; + case 24: + packBits24(values, ptr); + break; + case 25: + packBits25(values, ptr); + break; + case 26: + packBits26(values, ptr); + break; + case 27: + packBits27(values, ptr); + break; + case 28: + packBits28(values, ptr); + break; + case 29: + packBits29(values, ptr); + break; + case 30: + packBits30(values, ptr); + break; + case 31: + packBits31(values, ptr); + break; + case 32: + packBits32(values, ptr); + break; + case 33: + packBits33(values, ptr); + break; + case 34: + packBits34(values, ptr); + break; + case 35: + packBits35(values, ptr); + break; + case 36: + packBits36(values, ptr); + break; + case 37: + packBits37(values, ptr); + break; + case 38: + packBits38(values, ptr); + break; + case 39: + packBits39(values, ptr); + break; + case 40: + packBits40(values, ptr); + break; + case 41: + packBits41(values, ptr); + break; + case 42: + packBits42(values, ptr); + break; + case 43: + packBits43(values, ptr); + break; + case 44: + packBits44(values, ptr); + break; + case 45: + packBits45(values, ptr); + break; + case 46: + packBits46(values, ptr); + break; + case 47: + packBits47(values, ptr); + break; + case 48: + packBits48(values, ptr); + break; + case 49: + packBits49(values, ptr); + break; + case 50: + packBits50(values, ptr); + break; + case 51: + packBits51(values, ptr); + break; + case 52: + packBits52(values, ptr); + break; + case 53: + packBits53(values, ptr); + break; + case 54: + packBits54(values, ptr); + break; + case 55: + packBits55(values, ptr); + break; + case 56: + packBits56(values, ptr); + break; + case 57: + packBits57(values, ptr); + break; + case 58: + packBits58(values, ptr); + break; + case 59: + packBits59(values, ptr); + break; + case 60: + packBits60(values, ptr); + break; + case 61: + packBits61(values, ptr); + break; + case 62: + packBits62(values, ptr); + break; + case 63: + packBits63(values, ptr); + break; + default: + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "wrong number of bits in packBitsBlock8: " + std::to_string(bits), + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); + } +} + +static inline void +unpackBitsBlock8(uint64_t* values, const uint8_t* ptr, uint8_t bits) { + switch (bits) { + case 1: + unpackBits1(values, ptr); + break; + case 2: + unpackBits2(values, ptr); + break; + case 3: + unpackBits3(values, ptr); + break; + case 4: + unpackBits4(values, ptr); + break; + case 5: + unpackBits5(values, ptr); + break; + case 6: + unpackBits6(values, ptr); + break; + case 7: + unpackBits7(values, ptr); + break; + case 8: + unpackBits8(values, ptr); + break; + case 9: + unpackBits9(values, ptr); + break; + case 10: + unpackBits10(values, ptr); + break; + case 11: + unpackBits11(values, ptr); + break; + case 12: + unpackBits12(values, ptr); + break; + case 13: + unpackBits13(values, ptr); + break; + case 14: + unpackBits14(values, ptr); + break; + case 15: + unpackBits15(values, ptr); + break; + case 16: + unpackBits16(values, ptr); + break; + case 17: + unpackBits17(values, ptr); + break; + case 18: + unpackBits18(values, ptr); + break; + case 19: + unpackBits19(values, ptr); + break; + case 20: + unpackBits20(values, ptr); + break; + case 21: + unpackBits21(values, ptr); + break; + case 22: + unpackBits22(values, ptr); + break; + case 23: + unpackBits23(values, ptr); + break; + case 24: + unpackBits24(values, ptr); + break; + case 25: + unpackBits25(values, ptr); + break; + case 26: + unpackBits26(values, ptr); + break; + case 27: + unpackBits27(values, ptr); + break; + case 28: + unpackBits28(values, ptr); + break; + case 29: + unpackBits29(values, ptr); + break; + case 30: + unpackBits30(values, ptr); + break; + case 31: + unpackBits31(values, ptr); + break; + case 32: + unpackBits32(values, ptr); + break; + case 33: + unpackBits33(values, ptr); + break; + case 34: + unpackBits34(values, ptr); + break; + case 35: + unpackBits35(values, ptr); + break; + case 36: + unpackBits36(values, ptr); + break; + case 37: + unpackBits37(values, ptr); + break; + case 38: + unpackBits38(values, ptr); + break; + case 39: + unpackBits39(values, ptr); + break; + case 40: + unpackBits40(values, ptr); + break; + case 41: + unpackBits41(values, ptr); + break; + case 42: + unpackBits42(values, ptr); + break; + case 43: + unpackBits43(values, ptr); + break; + case 44: + unpackBits44(values, ptr); + break; + case 45: + unpackBits45(values, ptr); + break; + case 46: + unpackBits46(values, ptr); + break; + case 47: + unpackBits47(values, ptr); + break; + case 48: + unpackBits48(values, ptr); + break; + case 49: + unpackBits49(values, ptr); + break; + case 50: + unpackBits50(values, ptr); + break; + case 51: + unpackBits51(values, ptr); + break; + case 52: + unpackBits52(values, ptr); + break; + case 53: + unpackBits53(values, ptr); + break; + case 54: + unpackBits54(values, ptr); + break; + case 55: + unpackBits55(values, ptr); + break; + case 56: + unpackBits56(values, ptr); + break; + case 57: + unpackBits57(values, ptr); + break; + case 58: + unpackBits58(values, ptr); + break; + case 59: + unpackBits59(values, ptr); + break; + case 60: + unpackBits60(values, ptr); + break; + case 61: + unpackBits61(values, ptr); + break; + case 62: + unpackBits62(values, ptr); + break; + case 63: + unpackBits63(values, ptr); + break; + default: + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "wrong number of bits in unpackBitsBlock8: " + std::to_string(bits), + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); + } +} + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/CMakeLists.txt b/velox/external/theta/CMakeLists.txt new file mode 100644 index 00000000000..9f6182f5e2f --- /dev/null +++ b/velox/external/theta/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +velox_add_library( + velox_common_theta + CompactThetaSketchParser.cpp + ThetaUpdateSketchBase.cpp + ThetaSketch.cpp + ThetaUnionBase.cpp + ThetaUnion.cpp) + +velox_link_libraries( + velox_common_theta + PUBLIC velox_memory + PRIVATE velox_exception) + +if(${VELOX_BUILD_TESTING}) + add_subdirectory(tests) +endif() diff --git a/velox/external/theta/CommonDefs.h b/velox/external/theta/CommonDefs.h new file mode 100644 index 00000000000..b5ba42dde07 --- /dev/null +++ b/velox/external/theta/CommonDefs.h @@ -0,0 +1,131 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace facebook::velox::common::theta { + +static const uint64_t DEFAULT_SEED = 9001; + +enum resizeFactor { X1 = 0, X2, X4, X8 }; + +template +using string = std::basic_string< + char, + std::char_traits, + typename std::allocator_traits::template rebind_alloc>; + +// common random declarations +namespace randomUtils { +static std::random_device rd; // possibly unsafe in MinGW with GCC < 9.2 +static thread_local std::mt19937_64 rand(rd()); +static thread_local std::uniform_real_distribution<> next_double(0.0, 1.0); +static thread_local std::uniform_int_distribution next_uint64( + 0, + UINT64_MAX); + +// thread-safe random bit +static thread_local std::independent_bits_engine + random_bit(static_cast( + std::chrono::system_clock::now().time_since_epoch().count() + + std::hash{}(std::this_thread::get_id()))); + +inline void overrideSeed(uint64_t s) { + rand.seed(s); +} +} // namespace randomUtils + +// utility function to hide unused compiler warning +// usually has no additional cost +template +void unused(T&&...) {} + +// common helping functions +// TODO: find a better place for them + +constexpr uint8_t log2(uint32_t n) { + return (n > 1) ? 1 + log2(n >> 1) : 0; +} + +constexpr uint8_t lgSizeFromCount(uint32_t n, double load_factor) { + return log2(n) + + ((n > static_cast((1 << (log2(n) + 1)) * load_factor)) ? 2 : 1); +} + +// stream helpers to hide casts +template +static inline T read(std::istream& is) { + T value; + is.read(reinterpret_cast(&value), sizeof(T)); + return value; +} + +template +static inline void read(std::istream& is, T* ptr, size_t size_bytes) { + is.read(reinterpret_cast(ptr), size_bytes); +} + +template +static inline void write(std::ostream& os, T value) { + os.write(reinterpret_cast(&value), sizeof(T)); +} + +template +static inline void write(std::ostream& os, const T* ptr, size_t size_bytes) { + os.write(reinterpret_cast(ptr), size_bytes); +} + +template +T byteswap(T value) { + char* ptr = static_cast(static_cast(&value)); + const int len = sizeof(T); + for (size_t i = 0; i < len / 2; ++i) { + std::swap(ptr[i], ptr[len - i - 1]); + } + return value; +} + +template +static inline T readBigEndian(std::istream& is) { + T value; + is.read(reinterpret_cast(&value), sizeof(T)); + return byteswap(value); +} + +// wrapper for iterators to implement operator-> returning temporary value +template +class returnValueHolder { + public: + returnValueHolder(T value) : value_(value) {} + const T* operator->() const { + return std::addressof(value_); + } + + private: + T value_; +}; + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/CompactThetaSketchParser.cpp b/velox/external/theta/CompactThetaSketchParser.cpp new file mode 100644 index 00000000000..1925f311d71 --- /dev/null +++ b/velox/external/theta/CompactThetaSketchParser.cpp @@ -0,0 +1,261 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#ifndef COMPACT_THETA_SKETCH_PARSER_CPP +#define COMPACT_THETA_SKETCH_PARSER_CPP + +#include "CompactThetaSketchParser.h" +#include "MurmurHash3.h" +#include "ThetaHelpers.h" + +#include +#include + +namespace facebook::velox::common::theta { + +template +auto CompactThetaSketchParser::parse( + const void* ptr, + size_t size, + uint64_t seed, + bool dump_on_error) -> CompactThetaSketchData { + checkMemorySize(ptr, size, 8, dump_on_error); + checker::checkSketchType( + reinterpret_cast(ptr)[COMPACT_SKETCH_TYPE_BYTE], + COMPACT_SKETCH_TYPE); + uint8_t serial_version = + reinterpret_cast(ptr)[COMPACT_SKETCH_SERIAL_VERSION_BYTE]; + switch (serial_version) { + case 4: { + // version 4 sketches are ordered and always have entries (single item in + // exact mode is v3) + const uint16_t seed_hash = + reinterpret_cast(ptr)[COMPACT_SKETCH_SEED_HASH_U16]; + checker::checkSeedHash(seed_hash, compute_seed_hash(seed)); + const bool has_theta = reinterpret_cast( + ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 1; + uint64_t theta = ThetaConstants::MAX_THETA; + if (has_theta) { + checkMemorySize(ptr, size, 16, dump_on_error); + theta = + reinterpret_cast(ptr)[COMPACT_SKETCH_V4_THETA_U64]; + } + const uint8_t num_entries_bytes = reinterpret_cast( + ptr)[COMPACT_SKETCH_V4_NUM_ENTRIES_BYTES_BYTE]; + size_t data_offset_bytes = has_theta + ? COMPACT_SKETCH_V4_PACKED_DATA_ESTIMATION_BYTE + : COMPACT_SKETCH_V4_PACKED_DATA_EXACT_BYTE; + checkMemorySize( + ptr, size, data_offset_bytes + num_entries_bytes, dump_on_error); + uint32_t num_entries = 0; + const uint8_t* num_entries_ptr = + reinterpret_cast(ptr) + data_offset_bytes; + for (unsigned i = 0; i < num_entries_bytes; ++i) { + num_entries |= (*num_entries_ptr++) << (i << 3); + } + data_offset_bytes += num_entries_bytes; + const uint8_t entry_bits = reinterpret_cast( + ptr)[COMPACT_SKETCH_V4_ENTRY_BITS_BYTE]; + const size_t expected_bits = entry_bits * num_entries; + const size_t expected_size_bytes = + data_offset_bytes + wholeBytesToHoldBits(expected_bits); + checkMemorySize(ptr, size, expected_size_bytes, dump_on_error); + return { + false, + true, + seed_hash, + num_entries, + theta, + reinterpret_cast(ptr) + data_offset_bytes, + entry_bits}; + } + case 3: { + uint64_t theta = ThetaConstants::MAX_THETA; + const uint16_t seed_hash = + reinterpret_cast(ptr)[COMPACT_SKETCH_SEED_HASH_U16]; + if (reinterpret_cast(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & + (1 << COMPACT_SKETCH_IS_EMPTY_FLAG)) { + return {true, true, seed_hash, 0, theta, nullptr, 64}; + } + checker::checkSeedHash(seed_hash, compute_seed_hash(seed)); + const bool has_theta = reinterpret_cast( + ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] > 2; + if (has_theta) { + checkMemorySize( + ptr, + size, + (COMPACT_SKETCH_THETA_U64 + 1) * sizeof(uint64_t), + dump_on_error); + theta = + reinterpret_cast(ptr)[COMPACT_SKETCH_THETA_U64]; + } + if (reinterpret_cast( + ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE] == 1) { + checkMemorySize(ptr, size, 16, dump_on_error); + return { + false, + true, + seed_hash, + 1, + theta, + reinterpret_cast(ptr) + + COMPACT_SKETCH_SINGLE_ENTRY_U64, + 64}; + } + const uint32_t num_entries = reinterpret_cast( + ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32]; + const size_t entries_start_u64 = has_theta + ? COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 + : COMPACT_SKETCH_ENTRIES_EXACT_U64; + const uint64_t* entries = + reinterpret_cast(ptr) + entries_start_u64; + const size_t expected_size_bytes = + (entries_start_u64 + num_entries) * sizeof(uint64_t); + checkMemorySize(ptr, size, expected_size_bytes, dump_on_error); + const bool is_ordered = + reinterpret_cast(ptr)[COMPACT_SKETCH_FLAGS_BYTE] & + (1 << COMPACT_SKETCH_IS_ORDERED_FLAG); + return {false, is_ordered, seed_hash, num_entries, theta, entries, 64}; + } + case 1: { + uint16_t seed_hash = compute_seed_hash(seed); + const uint32_t num_entries = reinterpret_cast( + ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32]; + uint64_t theta = + reinterpret_cast(ptr)[COMPACT_SKETCH_THETA_U64]; + bool is_empty = + (num_entries == 0) && (theta == ThetaConstants::MAX_THETA); + if (is_empty) + return {true, true, seed_hash, 0, theta, nullptr, 64}; + const uint64_t* entries = reinterpret_cast(ptr) + + COMPACT_SKETCH_ENTRIES_ESTIMATION_U64; + const size_t expected_size_bytes = + (COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 + num_entries) * + sizeof(uint64_t); + checkMemorySize(ptr, size, expected_size_bytes, dump_on_error); + return {false, true, seed_hash, num_entries, theta, entries, 64}; + } + case 2: { + uint8_t preamble_size = + reinterpret_cast(ptr)[COMPACT_SKETCH_PRE_LONGS_BYTE]; + const uint16_t seed_hash = + reinterpret_cast(ptr)[COMPACT_SKETCH_SEED_HASH_U16]; + checker::checkSeedHash(seed_hash, compute_seed_hash(seed)); + if (preamble_size == 1) { + return { + true, true, seed_hash, 0, ThetaConstants::MAX_THETA, nullptr, 64}; + } else if (preamble_size == 2) { + const uint32_t num_entries = reinterpret_cast( + ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32]; + if (num_entries == 0) { + return { + true, true, seed_hash, 0, ThetaConstants::MAX_THETA, nullptr, 64}; + } else { + const size_t expected_size_bytes = (preamble_size + num_entries) << 3; + checkMemorySize(ptr, size, expected_size_bytes, dump_on_error); + const uint64_t* entries = reinterpret_cast(ptr) + + COMPACT_SKETCH_ENTRIES_EXACT_U64; + return { + false, + true, + seed_hash, + num_entries, + ThetaConstants::MAX_THETA, + entries, + 64}; + } + } else if (preamble_size == 3) { + const uint32_t num_entries = reinterpret_cast( + ptr)[COMPACT_SKETCH_NUM_ENTRIES_U32]; + uint64_t theta = + reinterpret_cast(ptr)[COMPACT_SKETCH_THETA_U64]; + bool is_empty = + (num_entries == 0) && (theta == ThetaConstants::MAX_THETA); + if (is_empty) + return {true, true, seed_hash, 0, theta, nullptr, 64}; + const uint64_t* entries = reinterpret_cast(ptr) + + COMPACT_SKETCH_ENTRIES_ESTIMATION_U64; + const size_t expected_size_bytes = + (COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 + num_entries) * + sizeof(uint64_t); + checkMemorySize(ptr, size, expected_size_bytes, dump_on_error); + return {false, true, seed_hash, num_entries, theta, entries, 64}; + } else { + throw VeloxUserError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + " longs of premable, but expected 1, 2, or 3", + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); + } + } + default: + throw VeloxUserError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "unsupported serial version " + std::to_string(serial_version), + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); + } +} + +template +void CompactThetaSketchParser::checkMemorySize( + const void* ptr, + size_t actual_bytes, + size_t expected_bytes, + bool dump_on_error) { + if (actual_bytes < expected_bytes) { + auto msg = "at least " + std::to_string(expected_bytes) + + " bytes expected, actual " + std::to_string(actual_bytes) + + (dump_on_error + ? (", sketch dump: " + + hexDump(reinterpret_cast(ptr), actual_bytes)) + : ""); + throw VeloxUserError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + msg, + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); + } +} + +template +std::string CompactThetaSketchParser::hexDump( + const uint8_t* ptr, + size_t size) { + std::stringstream s; + s << std::hex << std::setfill('0') << std::uppercase; + for (size_t i = 0; i < size; ++i) + s << std::setw(2) << (ptr[i] & 0xff); + return s.str(); +} + +} // namespace facebook::velox::common::theta + +#endif diff --git a/velox/external/theta/CompactThetaSketchParser.h b/velox/external/theta/CompactThetaSketchParser.h new file mode 100644 index 00000000000..64867ed61c5 --- /dev/null +++ b/velox/external/theta/CompactThetaSketchParser.h @@ -0,0 +1,82 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches +#pragma once + +#include + +namespace facebook::velox::common::theta { + +template +T wholeBytesToHoldBits(T bits) { + static_assert(std::is_integral::value, "integral type expected"); + return (bits >> 3) + ((bits & 7) > 0); +} + +template +class CompactThetaSketchParser { + public: + struct CompactThetaSketchData { + bool isEmpty; + bool isOrdered; + uint16_t seedHash; + uint32_t numEntries; + uint64_t theta; + const void* entriesStartPtr; + uint8_t entryBits; + }; + + static CompactThetaSketchData parse( + const void* ptr, + size_t size, + uint64_t seed, + bool dump_on_error = false); + + private: + // offsets are in sizeof(type) + static const size_t COMPACT_SKETCH_PRE_LONGS_BYTE = 0; + static const size_t COMPACT_SKETCH_SERIAL_VERSION_BYTE = 1; + static const size_t COMPACT_SKETCH_TYPE_BYTE = 2; + static const size_t COMPACT_SKETCH_FLAGS_BYTE = 5; + static const size_t COMPACT_SKETCH_SEED_HASH_U16 = 3; + static const size_t COMPACT_SKETCH_SINGLE_ENTRY_U64 = 1; // ver 3 + static const size_t COMPACT_SKETCH_NUM_ENTRIES_U32 = 2; // ver 1-3 + static const size_t COMPACT_SKETCH_ENTRIES_EXACT_U64 = 2; // ver 1-3 + static const size_t COMPACT_SKETCH_ENTRIES_ESTIMATION_U64 = 3; // ver 1-3 + static const size_t COMPACT_SKETCH_THETA_U64 = 2; // ver 1-3 + static const size_t COMPACT_SKETCH_V4_ENTRY_BITS_BYTE = 3; + static const size_t COMPACT_SKETCH_V4_NUM_ENTRIES_BYTES_BYTE = 4; + static const size_t COMPACT_SKETCH_V4_THETA_U64 = 1; + static const size_t COMPACT_SKETCH_V4_PACKED_DATA_EXACT_BYTE = 8; + static const size_t COMPACT_SKETCH_V4_PACKED_DATA_ESTIMATION_BYTE = 16; + + static const uint8_t COMPACT_SKETCH_IS_EMPTY_FLAG = 2; + static const uint8_t COMPACT_SKETCH_IS_ORDERED_FLAG = 4; + + static const uint8_t COMPACT_SKETCH_TYPE = 3; + + static void checkMemorySize( + const void* ptr, + size_t actual_bytes, + size_t expected_bytes, + bool dump_on_error); + static std::string hexDump(const uint8_t* ptr, size_t size); +}; + +} // namespace facebook::velox::common::theta + +#include "CompactThetaSketchParser.cpp" diff --git a/velox/external/theta/ConditionalForward.h b/velox/external/theta/ConditionalForward.h new file mode 100644 index 00000000000..f90e920fddb --- /dev/null +++ b/velox/external/theta/ConditionalForward.h @@ -0,0 +1,81 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include +#include + +namespace facebook::velox::common::theta { + +// Forward type T2 as rvalue reference if type T1 is rvalue reference + +template +using fwdType = typename std::conditional< + std::is_lvalue_reference::value, + T2, + typename std::remove_reference::type&&>::type; + +template +fwdType conditionalForward(T2&& value) { + return std::forward>(std::forward(value)); +} + +// Forward container as iterators + +template +auto forwardBegin(Container&& c) -> typename std::enable_if< + std::is_lvalue_reference::value || + std::is_same< + typename std::remove_reference::type::const_iterator, + decltype(c.begin())>::value, + decltype(c.begin())>::type { + return c.begin(); +} + +template +auto forwardBegin(Container&& c) -> typename std::enable_if< + !std::is_lvalue_reference::value && + !std::is_same< + typename std::remove_reference::type::const_iterator, + decltype(c.begin())>::value, + decltype(std::make_move_iterator(c.begin()))>::type { + return std::make_move_iterator(c.begin()); +} + +template +auto forwardEnd(Container&& c) -> typename std::enable_if< + std::is_lvalue_reference::value || + std::is_same< + typename std::remove_reference::type::const_iterator, + decltype(c.begin())>::value, + decltype(c.end())>::type { + return c.end(); +} + +template +auto forwardEnd(Container&& c) -> typename std::enable_if< + !std::is_lvalue_reference::value && + !std::is_same< + typename std::remove_reference::type::const_iterator, + decltype(c.begin())>::value, + decltype(std::make_move_iterator(c.end()))>::type { + return std::make_move_iterator(c.end()); +} + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/CountZeros.h b/velox/external/theta/CountZeros.h new file mode 100644 index 00000000000..c898b16a52d --- /dev/null +++ b/velox/external/theta/CountZeros.h @@ -0,0 +1,109 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include + +namespace facebook::velox::common::theta { + +static const uint8_t byteLeadingZerosTable[256] = { + 8, 7, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + +static const uint8_t byteTrailingZerosTable[256] = { + 8, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, + 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 6, 0, 1, 0, 2, 0, 1, 0, + 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, + 3, 0, 1, 0, 2, 0, 1, 0, 7, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, + 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 6, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, 4, 0, 1, 0, 2, 0, 1, 0, + 3, 0, 1, 0, 2, 0, 1, 0, 5, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0, + 4, 0, 1, 0, 2, 0, 1, 0, 3, 0, 1, 0, 2, 0, 1, 0}; + +static const uint64_t FCLZ_MASK_56 = 0x00ffffffffffffff; +static const uint64_t FCLZ_MASK_48 = 0x0000ffffffffffff; +static const uint64_t FCLZ_MASK_40 = 0x000000ffffffffff; +static const uint64_t FCLZ_MASK_32 = 0x00000000ffffffff; +static const uint64_t FCLZ_MASK_24 = 0x0000000000ffffff; +static const uint64_t FCLZ_MASK_16 = 0x000000000000ffff; +static const uint64_t FCLZ_MASK_08 = 0x00000000000000ff; + +static inline uint8_t countLeadingZerosInU64(uint64_t input) { + if (input > FCLZ_MASK_56) + return byteLeadingZerosTable[(input >> 56) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_48) + return 8 + byteLeadingZerosTable[(input >> 48) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_40) + return 16 + byteLeadingZerosTable[(input >> 40) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_32) + return 24 + byteLeadingZerosTable[(input >> 32) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_24) + return 32 + byteLeadingZerosTable[(input >> 24) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_16) + return 40 + byteLeadingZerosTable[(input >> 16) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_08) + return 48 + byteLeadingZerosTable[(input >> 8) & FCLZ_MASK_08]; + if (true) + return 56 + byteLeadingZerosTable[(input)&FCLZ_MASK_08]; +} + +static inline uint8_t countLeadingZerosInU32(uint32_t input) { + if (input > FCLZ_MASK_24) + return byteLeadingZerosTable[(input >> 24) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_16) + return 8 + byteLeadingZerosTable[(input >> 16) & FCLZ_MASK_08]; + if (input > FCLZ_MASK_08) + return 16 + byteLeadingZerosTable[(input >> 8) & FCLZ_MASK_08]; + if (true) + return 24 + byteLeadingZerosTable[(input)&FCLZ_MASK_08]; +} + +static inline uint8_t countTrailingZerosInU32(uint32_t input) { + for (int i = 0; i < 4; i++) { + const int byte = input & 0xff; + if (byte != 0) + return static_cast((i << 3) + byteLeadingZerosTable[byte]); + input >>= 8; + } + return 32; +} + +static inline uint8_t countTrailingZerosInU64(uint64_t input) { + for (int i = 0; i < 8; i++) { + const int byte = input & 0xff; + if (byte != 0) + return static_cast((i << 3) + byteLeadingZerosTable[byte]); + input >>= 8; + } + return 64; +} + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/MemoryOperations.h b/velox/external/theta/MemoryOperations.h new file mode 100644 index 00000000000..81c2e972b3f --- /dev/null +++ b/velox/external/theta/MemoryOperations.h @@ -0,0 +1,90 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include "velox/common/base/Exceptions.h" +#include +#include +#include +#include +#include + +namespace facebook::velox::common::theta { + +static inline void ensureMinimumMemory( + size_t bytes_available, + size_t min_needed) { + if (bytes_available < min_needed) { + auto msg = "Insufficient buffer size detected: bytes available " + + std::to_string(bytes_available) + ", minimum needed " + + std::to_string(min_needed); + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + msg, + error_source::kErrorSourceUser, + error_code::kGenericUserError, + false /*retriable*/); + } +} + +static inline void checkMemorySize(size_t requested_index, size_t capacity) { + if (requested_index > capacity) { + auto msg = "Attempt to access memory beyond limits: requested index " + + std::to_string(requested_index) + ", capacity " + + std::to_string(capacity); + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + msg, + error_source::kErrorSourceUser, + error_code::kGenericUserError, + false /*retriable*/); + } +} + +// note: size is in bytes, not items +static inline size_t copyFromMem(const void* src, void* dst, size_t size) { + memcpy(dst, src, size); + return size; +} + +// note: size is in bytes, not items +static inline size_t copyToMem(const void* src, void* dst, size_t size) { + memcpy(dst, src, size); + return size; +} + +template +static inline size_t copyFromMem(const void* src, T& item) { + memcpy(&item, src, sizeof(T)); + return sizeof(T); +} + +template +static inline size_t copyToMem(T item, void* dst) { + memcpy(dst, &item, sizeof(T)); + return sizeof(T); +} + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/MurmurHash3.h b/velox/external/theta/MurmurHash3.h new file mode 100644 index 00000000000..bc5dd67fd76 --- /dev/null +++ b/velox/external/theta/MurmurHash3.h @@ -0,0 +1,221 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +// Minimally modified from Austin Applebee's code: +// * Removed MurmurHash3_x86_32 and MurmurHash3_x86_128 +// * Changed input seed in MurmurHash3_x64_128 to uint64_t +// * Define and use HashState reference to return result +// * Made entire hash function defined inline +// * Added compute_seed_hash +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +// Note - The x86 and x64 versions do _not_ produce the same results, as the +// algorithms are optimized for their respective platforms. You can still +// compile and run any of them on any platform, but your performance with the +// non-native version will be less than optimal. + +#pragma once + +#include + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) + +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef unsigned __int64 uint64_t; + +#define MURMUR3_FORCE_INLINE __forceinline + +#include + +#define MURMUR3_ROTL64(x, y) _rotl64(x, y) + +#define MURMUR3_BIG_CONSTANT(x) (x) + +// Other compilers + +#else // defined(_MSC_VER) + +#include + +#define MURMUR3_FORCE_INLINE inline __attribute__((always_inline)) + +inline uint64_t rotl64(uint64_t x, int8_t r) { + return (x << r) | (x >> (64 - r)); +} + +#define MURMUR3_ROTL64(x, y) rotl64(x, y) + +#define MURMUR3_BIG_CONSTANT(x) (x##LLU) + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- + +//----------------------------------------------------------------------------- +// Return type - Using C++ reference for return type which should allow better +// compiler optimization than a void* pointer +typedef struct { + uint64_t h1; + uint64_t h2; +} HashState; + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +MURMUR3_FORCE_INLINE uint64_t getblock64(const uint8_t* p, size_t i) { + uint64_t res; + memcpy(&res, p + i * sizeof(uint64_t), sizeof(res)); + return res; +} + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +MURMUR3_FORCE_INLINE uint64_t fmix64(uint64_t k) { + k ^= k >> 33; + k *= MURMUR3_BIG_CONSTANT(0xff51afd7ed558ccd); + k ^= k >> 33; + k *= MURMUR3_BIG_CONSTANT(0xc4ceb9fe1a85ec53); + k ^= k >> 33; + + return k; +} + +MURMUR3_FORCE_INLINE void MurmurHash3_x64_128( + const void* key, + size_t lenBytes, + uint64_t seed, + HashState& out) { + static const uint64_t c1 = MURMUR3_BIG_CONSTANT(0x87c37b91114253d5); + static const uint64_t c2 = MURMUR3_BIG_CONSTANT(0x4cf5ad432745937f); + + const uint8_t* data = (const uint8_t*)key; + + out.h1 = seed; + out.h2 = seed; + + // Number of full 128-bit blocks of 16 bytes. + // Possible exclusion of a remainder of up to 15 bytes. + const size_t nblocks = lenBytes >> 4; // bytes / 16 + + // Process the 128-bit blocks (the body) into the hash + for (size_t i = 0; i < nblocks; ++i) { // 16 bytes per block + uint64_t k1 = getblock64(data, i * 2 + 0); + uint64_t k2 = getblock64(data, i * 2 + 1); + + k1 *= c1; + k1 = MURMUR3_ROTL64(k1, 31); + k1 *= c2; + out.h1 ^= k1; + out.h1 = MURMUR3_ROTL64(out.h1, 27); + out.h1 += out.h2; + out.h1 = out.h1 * 5 + 0x52dce729; + + k2 *= c2; + k2 = MURMUR3_ROTL64(k2, 33); + k2 *= c1; + out.h2 ^= k2; + out.h2 = MURMUR3_ROTL64(out.h2, 31); + out.h2 += out.h1; + out.h2 = out.h2 * 5 + 0x38495ab5; + } + + // tail + const uint8_t* tail = (const uint8_t*)(data + (nblocks << 4)); + + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch (lenBytes & 15) { + case 15: + k2 ^= ((uint64_t)tail[14]) << 48; // falls through + case 14: + k2 ^= ((uint64_t)tail[13]) << 40; // falls through + case 13: + k2 ^= ((uint64_t)tail[12]) << 32; // falls through + case 12: + k2 ^= ((uint64_t)tail[11]) << 24; // falls through + case 11: + k2 ^= ((uint64_t)tail[10]) << 16; // falls through + case 10: + k2 ^= ((uint64_t)tail[9]) << 8; // falls through + case 9: + k2 ^= ((uint64_t)tail[8]) << 0; + k2 *= c2; + k2 = MURMUR3_ROTL64(k2, 33); + k2 *= c1; + out.h2 ^= k2; + // falls through + case 8: + k1 ^= ((uint64_t)tail[7]) << 56; // falls through + case 7: + k1 ^= ((uint64_t)tail[6]) << 48; // falls through + case 6: + k1 ^= ((uint64_t)tail[5]) << 40; // falls through + case 5: + k1 ^= ((uint64_t)tail[4]) << 32; // falls through + case 4: + k1 ^= ((uint64_t)tail[3]) << 24; // falls through + case 3: + k1 ^= ((uint64_t)tail[2]) << 16; // falls through + case 2: + k1 ^= ((uint64_t)tail[1]) << 8; // falls through + case 1: + k1 ^= ((uint64_t)tail[0]) << 0; + k1 *= c1; + k1 = MURMUR3_ROTL64(k1, 31); + k1 *= c2; + out.h1 ^= k1; + }; + + //---------- + // finalization + + out.h1 ^= lenBytes; + out.h2 ^= lenBytes; + + out.h1 += out.h2; + out.h2 += out.h1; + + out.h1 = fmix64(out.h1); + out.h2 = fmix64(out.h2); + + out.h1 += out.h2; + out.h2 += out.h1; +} + +//----------------------------------------------------------------------------- + +MURMUR3_FORCE_INLINE uint16_t compute_seed_hash(uint64_t seed) { + HashState hashes; + MurmurHash3_x64_128(&seed, sizeof(seed), 0, hashes); + return static_cast(hashes.h1 & 0xffff); +} + +#undef MURMUR3_FORCE_INLINE +#undef MURMUR3_ROTL64 +#undef MURMUR3_BIG_CONSTANT diff --git a/velox/external/theta/Serde.h b/velox/external/theta/Serde.h new file mode 100644 index 00000000000..3215e62a866 --- /dev/null +++ b/velox/external/theta/Serde.h @@ -0,0 +1,297 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include +#include +#include +#include +#include + +#include "MemoryOperations.h" +#include "velox/common/base/Exceptions.h" + +namespace facebook::velox::common::theta { + +/// Interface for serializing and deserializing items +template +struct serde { + /** + * Stream serialization + * @param os output stream + * @param items pointer to array of items + * @param num number of items + */ + void serialize(std::ostream& os, const T* items, unsigned num) const; + + /** + * Stream deserialization + * @param is input stream + * @param items pointer to array of items (items in the array are allocated + * but not initialized) + * @param num number of items + */ + void deserialize(std::istream& is, T* items, unsigned num) const; + + /** + * Raw bytes serialization + * @param ptr pointer to output buffer + * @param capacity size of the buffer in bytes + * @param items pointer to array of items + * @param num number of items + */ + size_t serialize(void* ptr, size_t capacity, const T* items, unsigned num) + const; + + /** + * Raw bytes deserialization + * @param ptr pointer to input buffer + * @param capacity size of the buffer in bytes + * @param items pointer to array of items (items in the array are allocated + * but not initialized) + * @param num number of items + */ + size_t deserialize(const void* ptr, size_t capacity, T* items, unsigned num) + const; + + /** + * Size of the given item + * @param item to be sized + * @return size of the given item in bytes + */ + size_t size_of_item(const T& item) const; +}; + +/// serde for all fixed-size arithmetic types (int and float of different +/// sizes). in particular, kll_sketch should produce sketches +/// binary-compatible with LongsSketch and ItemsSketch with +/// ArrayOfLongsSerDe in Java +template +struct serde::value>::type> { + /// @copydoc serde::serialize + void serialize(std::ostream& os, const T* items, unsigned num) const { + bool failure = false; + try { + os.write(reinterpret_cast(items), sizeof(T) * num); + } catch (std::ostream::failure&) { + failure = true; + } + if (failure || !os.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error writing to std::ostream with " + std::to_string(num) + + " items", + error_source::kErrorSourceUser, + error_code::kGenericUserError, + false /*retriable*/); + } + } + + void deserialize(std::istream& is, T* items, unsigned num) const { + bool failure = false; + try { + is.read((char*)items, sizeof(T) * num); + } catch (std::istream::failure&) { + failure = true; + } + if (failure || !is.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error reading from std::istream with " + std::to_string(num) + + " items", + error_source::kErrorSourceUser, + error_code::kGenericUserError, + false /*retriable*/); + } + } + + /// @copydoc serde::serialize(void*,size_t,const T*,unsigned) const + size_t serialize(void* ptr, size_t capacity, const T* items, unsigned num) + const { + const size_t bytes_written = sizeof(T) * num; + checkMemorySize(bytes_written, capacity); + memcpy(ptr, items, bytes_written); + return bytes_written; + } + + /// @copydoc serde::deserialize(const void*,size_t,T*,unsigned) const + size_t deserialize(const void* ptr, size_t capacity, T* items, unsigned num) + const { + const size_t bytes_read = sizeof(T) * num; + checkMemorySize(bytes_read, capacity); + memcpy(items, ptr, bytes_read); + return bytes_read; + } + + /// @copydoc serde::size_of_item + size_t size_of_item(const T& item) const { + unused(item); + return sizeof(T); + } +}; + +/// serde for std::string items. +/// This should produce sketches binary-compatible with +/// ItemsSketch with ArrayOfStringsSerDe in Java. +/// The length of each string is stored as a 32-bit integer (historically), +/// which may be too wasteful. Treat this as an example. +template <> +struct serde { + /// @copydoc serde::serialize + void serialize(std::ostream& os, const std::string* items, unsigned num) + const { + unsigned i = 0; + bool failure = false; + try { + for (; i < num && os.good(); i++) { + uint32_t length = static_cast(items[i].size()); + os.write((char*)&length, sizeof(length)); + os.write(items[i].c_str(), length); + } + } catch (std::ostream::failure&) { + failure = true; + } + if (failure || !os.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error writing to std::ostream at item " + std::to_string(i), + error_source::kErrorSourceUser, + error_code::kGenericUserError, + false /*retriable*/); + } + } + + /// @copydoc serde::deserialize + void deserialize(std::istream& is, std::string* items, unsigned num) const { + unsigned i = 0; + bool failure = false; + try { + for (; i < num; i++) { + uint32_t length; + is.read((char*)&length, sizeof(length)); + if (!is.good()) { + break; + } + std::string str; + str.reserve(length); + for (uint32_t j = 0; j < length; j++) { + str.push_back(static_cast(is.get())); + } + if (!is.good()) { + break; + } + new (&items[i]) std::string(std::move(str)); + } + } catch (std::istream::failure&) { + failure = true; + } + if (failure || !is.good()) { + // clean up what we've already allocated + for (unsigned j = 0; j < i; ++j) { + items[j].~basic_string(); + } + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error reading from std::istream at item " + std::to_string(i), + error_source::kErrorSourceUser, + error_code::kGenericUserError, + false /*retriable*/); + } + } + + /// @copydoc serde::serialize(void*,size_t,const T*,unsigned) const + size_t serialize( + void* ptr, + size_t capacity, + const std::string* items, + unsigned num) const { + size_t bytes_written = 0; + for (unsigned i = 0; i < num; ++i) { + const uint32_t length = static_cast(items[i].size()); + const size_t new_bytes = length + sizeof(length); + checkMemorySize(bytes_written + new_bytes, capacity); + memcpy(ptr, &length, sizeof(length)); + ptr = static_cast(ptr) + sizeof(uint32_t); + memcpy(ptr, items[i].c_str(), length); + ptr = static_cast(ptr) + length; + bytes_written += new_bytes; + } + return bytes_written; + } + + /// @copydoc serde::deserialize(const void*,size_t,T*,unsigned) const + size_t deserialize( + const void* ptr, + size_t capacity, + std::string* items, + unsigned num) const { + size_t bytes_read = 0; + unsigned i = 0; + bool failure = false; + for (; i < num && !failure; ++i) { + uint32_t length; + if (bytes_read + sizeof(length) > capacity) { + bytes_read += sizeof(length); // we'll use this to report the error + failure = true; + break; + } + memcpy(&length, ptr, sizeof(length)); + ptr = static_cast(ptr) + sizeof(uint32_t); + bytes_read += sizeof(length); + + if (bytes_read + length > capacity) { + bytes_read += length; // we'll use this to report the error + failure = true; + break; + } + new (&items[i]) std::string(static_cast(ptr), length); + ptr = static_cast(ptr) + length; + bytes_read += length; + } + + if (failure) { + // clean up what we've already allocated + for (unsigned j = 0; j < i; ++j) + items[j].~basic_string(); + // using this for a consistent error message + checkMemorySize(bytes_read, capacity); + } + + return bytes_read; + } + + /// @copydoc serde::size_of_item + size_t size_of_item(const std::string& item) const { + return sizeof(uint32_t) + item.size(); + } +}; + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/ThetaComparators.h b/velox/external/theta/ThetaComparators.h new file mode 100644 index 00000000000..e35eb96578c --- /dev/null +++ b/velox/external/theta/ThetaComparators.h @@ -0,0 +1,48 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include + +namespace facebook::velox::common::theta { + +template +struct compareByKey { + template + bool operator()(Entry1&& a, Entry2&& b) const { + return ExtractKey()(std::forward(a)) < + ExtractKey()(std::forward(b)); + } +}; + +// less than + +template +class KeyLessThan { + public: + explicit KeyLessThan(const Key& key) : key(key) {} + bool operator()(const Entry& entry) const { + return ExtractKey()(entry) < this->key; + } + + private: + Key key; +}; + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/ThetaConstants.h b/velox/external/theta/ThetaConstants.h new file mode 100644 index 00000000000..e59a2b6c61f --- /dev/null +++ b/velox/external/theta/ThetaConstants.h @@ -0,0 +1,43 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include +#include "CommonDefs.h" + +namespace facebook::velox::common::theta { + +/// Theta constants +namespace ThetaConstants { +/// hash table resize factor +using resizeFactor = facebook::velox::common::theta::resizeFactor; +/// default resize factor +const resizeFactor DEFAULT_RESIZE_FACTOR = resizeFactor::X8; + +/// max theta - signed max for compatibility with Java +const uint64_t MAX_THETA = LLONG_MAX; +/// min log2 of K +const uint8_t MIN_LG_K = 5; +/// max log2 of K +const uint8_t MAX_LG_K = 26; +/// default log2 of K +const uint8_t DEFAULT_LG_K = 12; +} // namespace ThetaConstants + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/ThetaHelpers.h b/velox/external/theta/ThetaHelpers.h new file mode 100644 index 00000000000..27c32f8cb10 --- /dev/null +++ b/velox/external/theta/ThetaHelpers.h @@ -0,0 +1,83 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include +#include + +#include "ThetaConstants.h" +#include "velox/common/base/Exceptions.h" + +namespace facebook::velox::common::theta { + +template +static void checkValue(T actual, T expected, const char* description) { + if (actual != expected) { + auto msg = std::string(description) + " mismatch: expected " + + std::to_string(expected) + ", actual " + std::to_string(actual); + throw VeloxUserError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + msg, + error_source::kErrorSourceUser, + error_code::kInvalidArgument, + false /*retriable*/); + } +} + +template +class checker { + public: + static void checkSerialVersion(uint8_t actual, uint8_t expected) { + checkValue(actual, expected, "serial version"); + } + static void checkSketchFamily(uint8_t actual, uint8_t expected) { + checkValue(actual, expected, "sketch family"); + } + static void checkSketchType(uint8_t actual, uint8_t expected) { + checkValue(actual, expected, "sketch type"); + } + static void checkSeedHash(uint16_t actual, uint16_t expected) { + checkValue(actual, expected, "seed hash"); + } +}; + +template +class ThetaBuildHelper { + public: + // consistent way of initializing theta from p + // avoids multiplication if p == 1 since it might not yield MAX_THETA exactly + static uint64_t startingThetaFromP(float p) { + if (p < 1) + return static_cast( + static_cast(ThetaConstants::MAX_THETA) * p); + return ThetaConstants::MAX_THETA; + } + + static uint8_t + startingSubMultiple(uint8_t lg_tgt, uint8_t lg_min, uint8_t lg_rf) { + return (lg_tgt <= lg_min) ? lg_min + : (lg_rf == 0) ? lg_tgt + : ((lg_tgt - lg_min) % lg_rf) + lg_min; + } +}; + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/ThetaSketch.cpp b/velox/external/theta/ThetaSketch.cpp new file mode 100644 index 00000000000..e72929268f7 --- /dev/null +++ b/velox/external/theta/ThetaSketch.cpp @@ -0,0 +1,1122 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#ifndef THETA_SKETCH_CPP +#define THETA_SKETCH_CPP + +#include +#include + +#include "BinomialBounds.h" +#include "BitPacking.h" +#include "CompactThetaSketchParser.h" +#include "CountZeros.h" +#include "MemoryOperations.h" +#include "ThetaSketch.h" +#include "velox/common/base/Exceptions.h" + +namespace facebook::velox::common::theta { + +template +bool BaseThetaSketchAlloc::isEstimationMode() const { + return getTheta64() < ThetaConstants::MAX_THETA && !isEmpty(); +} + +template +double BaseThetaSketchAlloc::getTheta() const { + return static_cast(getTheta64()) / + static_cast(ThetaConstants::MAX_THETA); +} + +template +double BaseThetaSketchAlloc::getEstimate() const { + return getNumRetained() / getTheta(); +} + +template +double BaseThetaSketchAlloc::getLowerBound(uint8_t num_std_devs) const { + if (!isEstimationMode()) + return getNumRetained(); + return BinomialBounds::getLowerBound( + getNumRetained(), getTheta(), num_std_devs); +} + +template +double BaseThetaSketchAlloc::getUpperBound(uint8_t num_std_devs) const { + if (!isEstimationMode()) + return getNumRetained(); + return BinomialBounds::getUpperBound( + getNumRetained(), getTheta(), num_std_devs); +} + +template +string BaseThetaSketchAlloc::toString(bool print_details) const { + // Using a temporary stream for implementation here does not comply with + // AllocatorAwareContainer requirements. The stream does not support passing + // an allocator instance, and alternatives are complicated. + std::ostringstream os; + os << "### Theta sketch summary:" << std::endl; + os << " num retained entries : " << this->getNumRetained() << std::endl; + os << " seed hash : " << this->getSeedHash() << std::endl; + os << " empty? : " << (this->isEmpty() ? "true" : "false") + << std::endl; + os << " ordered? : " << (this->isOrdered() ? "true" : "false") + << std::endl; + os << " estimation mode? : " + << (this->isEstimationMode() ? "true" : "false") << std::endl; + os << " theta (fraction) : " << this->getTheta() << std::endl; + os << " theta (raw 64-bit) : " << this->getTheta64() << std::endl; + os << " estimate : " << this->getEstimate() << std::endl; + os << " lower bound 95% conf : " << this->getLowerBound(2) << std::endl; + os << " upper bound 95% conf : " << this->getUpperBound(2) << std::endl; + printSpecifics(os); + os << "### End sketch summary" << std::endl; + if (print_details) { + printItems(os); + } + return string(os.str().c_str(), this->getAllocator()); +} + +template +void ThetaSketchAlloc::printItems(std::ostringstream& os) const { + os << "### Retained entries" << std::endl; + for (const auto& hash : *this) { + os << hash << std::endl; + } + os << "### End retained entries" << std::endl; +} + +// update sketch + +template +UpdateThetaSketchAlloc::UpdateThetaSketchAlloc( + uint8_t lgCurSize, + uint8_t lgNomSize, + resizeFactor rf, + float p, + uint64_t theta, + uint64_t seed, + const A& allocator) + : table_(lgCurSize, lgNomSize, rf, p, theta, seed, allocator) {} + +template +A UpdateThetaSketchAlloc::getAllocator() const { + return table_.allocator_; +} + +template +bool UpdateThetaSketchAlloc::isEmpty() const { + return table_.isEmpty_; +} + +template +bool UpdateThetaSketchAlloc::isOrdered() const { + return table_.numEntries_ > 1 ? false : true; +} + +template +uint64_t UpdateThetaSketchAlloc::getTheta64() const { + return isEmpty() ? ThetaConstants::MAX_THETA : table_.theta_; +} + +template +uint32_t UpdateThetaSketchAlloc::getNumRetained() const { + return table_.numEntries_; +} + +template +uint16_t UpdateThetaSketchAlloc::getSeedHash() const { + return compute_seed_hash(table_.seed_); +} + +template +uint8_t UpdateThetaSketchAlloc::getLgK() const { + return table_.lgNomSize_; +} + +template +auto UpdateThetaSketchAlloc::getRf() const -> resizeFactor { + return table_.rf_; +} + +template +void UpdateThetaSketchAlloc::update(uint64_t value) { + update(&value, sizeof(value)); +} + +template +void UpdateThetaSketchAlloc::update(int64_t value) { + update(&value, sizeof(value)); +} + +template +void UpdateThetaSketchAlloc::update(uint32_t value) { + update(static_cast(value)); +} + +template +void UpdateThetaSketchAlloc::update(int32_t value) { + update(static_cast(value)); +} + +template +void UpdateThetaSketchAlloc::update(uint16_t value) { + update(static_cast(value)); +} + +template +void UpdateThetaSketchAlloc::update(int16_t value) { + update(static_cast(value)); +} + +template +void UpdateThetaSketchAlloc::update(uint8_t value) { + update(static_cast(value)); +} + +template +void UpdateThetaSketchAlloc::update(int8_t value) { + update(static_cast(value)); +} + +template +void UpdateThetaSketchAlloc::update(double value) { + update(canonical_double(value)); +} + +template +void UpdateThetaSketchAlloc::update(float value) { + update(static_cast(value)); +} + +template +void UpdateThetaSketchAlloc::update(const std::string& value) { + if (value.empty()) + return; + update(value.c_str(), value.length()); +} + +template +void UpdateThetaSketchAlloc::update(const void* data, size_t length) { + const uint64_t hash = table_.hashAndScreen(data, length); + if (hash == 0) + return; + auto result = table_.find(hash); + if (!result.second) { + table_.insert(result.first, hash); + } +} + +template +void UpdateThetaSketchAlloc::trim() { + table_.trim(); +} + +template +void UpdateThetaSketchAlloc::reset() { + table_.reset(); +} + +template +auto UpdateThetaSketchAlloc::begin() -> iterator { + return iterator(table_.entries_, 1 << table_.lgCurSize_, 0); +} + +template +auto UpdateThetaSketchAlloc::end() -> iterator { + return iterator(nullptr, 0, 1 << table_.lgCurSize_); +} + +template +auto UpdateThetaSketchAlloc::begin() const -> const_iterator { + return const_iterator(table_.entries_, 1 << table_.lgCurSize_, 0); +} + +template +auto UpdateThetaSketchAlloc::end() const -> const_iterator { + return const_iterator(nullptr, 0, 1 << table_.lgCurSize_); +} + +template +CompactThetaSketchAlloc UpdateThetaSketchAlloc::compact( + bool ordered) const { + return CompactThetaSketchAlloc(*this, ordered); +} + +template +void UpdateThetaSketchAlloc::printSpecifics(std::ostringstream& os) const { + os << " lg nominal size : " << static_cast(table_.lgNomSize_) + << std::endl; + os << " lg current size : " << static_cast(table_.lgCurSize_) + << std::endl; + os << " resize factor : " << (1 << table_.rf_) << std::endl; +} + +// builder + +template +UpdateThetaSketchAlloc::builder::builder(const A& allocator) + : ThetaBaseBuilder(allocator) {} + +template +UpdateThetaSketchAlloc UpdateThetaSketchAlloc::builder::build() const { + return UpdateThetaSketchAlloc( + this->startingLgSize(), + this->lg_k_, + this->rf_, + this->p_, + this->startingTheta(), + this->seed_, + this->allocator_); +} + +// compact sketch + +template +template +CompactThetaSketchAlloc::CompactThetaSketchAlloc( + const Other& other, + bool ordered) + : isEmpty_(other.isEmpty()), + isOrdered_(other.isOrdered() || ordered), + seedHash_(other.getSeedHash()), + theta_(other.getTheta64()), + entries_(other.getAllocator()) { + if (!other.isEmpty()) { + entries_.reserve(other.getNumRetained()); + std::copy(other.begin(), other.end(), std::back_inserter(entries_)); + if (ordered && !other.isOrdered()) + std::sort(entries_.begin(), entries_.end()); + } +} + +template +CompactThetaSketchAlloc::CompactThetaSketchAlloc( + bool isEmpty, + bool isOrdered, + uint16_t seedHash, + uint64_t theta, + std::vector&& entries) + : isEmpty_(isEmpty), + isOrdered_(isOrdered || (entries.size() <= 1ULL)), + seedHash_(seedHash), + theta_(theta), + entries_(std::move(entries)) {} + +template +A CompactThetaSketchAlloc::getAllocator() const { + return entries_.get_allocator(); +} + +template +bool CompactThetaSketchAlloc::isEmpty() const { + return isEmpty_; +} + +template +bool CompactThetaSketchAlloc::isOrdered() const { + return isOrdered_; +} + +template +uint64_t CompactThetaSketchAlloc::getTheta64() const { + return theta_; +} + +template +uint32_t CompactThetaSketchAlloc::getNumRetained() const { + return static_cast(entries_.size()); +} + +template +uint16_t CompactThetaSketchAlloc::getSeedHash() const { + return seedHash_; +} + +template +auto CompactThetaSketchAlloc::begin() -> iterator { + return iterator(entries_.data(), static_cast(entries_.size()), 0); +} + +template +auto CompactThetaSketchAlloc::end() -> iterator { + return iterator(nullptr, 0, static_cast(entries_.size())); +} + +template +auto CompactThetaSketchAlloc::begin() const -> const_iterator { + return const_iterator( + entries_.data(), static_cast(entries_.size()), 0); +} + +template +auto CompactThetaSketchAlloc::end() const -> const_iterator { + return const_iterator(nullptr, 0, static_cast(entries_.size())); +} + +template +void CompactThetaSketchAlloc::printSpecifics(std::ostringstream&) const {} + +template +uint8_t CompactThetaSketchAlloc::getPreambleLongs(bool compressed) const { + if (compressed) { + return this->isEstimationMode() ? 2 : 1; + } + return this->isEstimationMode() ? 3 + : this->isEmpty() || entries_.size() == 1 ? 1 + : 2; +} + +template +size_t CompactThetaSketchAlloc::getMaxSerializedSizeBytes(uint8_t lg_k) { + return sizeof(uint64_t) * + (3 + UpdateThetaSketchAlloc::ThetaTable::getCapacity(lg_k + 1, lg_k)); +} + +template +size_t CompactThetaSketchAlloc::getSerializedSizeBytes( + bool compressed) const { + if (compressed && isSuitableForCompression()) { + return getCompressedSerializedSizeBytes( + computeEntryBits(), getNumEntriesBytes()); + } + return sizeof(uint64_t) * getPreambleLongs(false) + + sizeof(uint64_t) * entries_.size(); +} + +// store num_entries as whole bytes since whole-byte blocks will follow (most +// probably) +template +uint8_t CompactThetaSketchAlloc::getNumEntriesBytes() const { + return wholeBytesToHoldBits( + 32 - countLeadingZerosInU32(static_cast(entries_.size()))); +} + +template +size_t CompactThetaSketchAlloc::getCompressedSerializedSizeBytes( + uint8_t entry_bits, + uint8_t num_entries_bytes) const { + const size_t compressed_bits = entry_bits * entries_.size(); + return sizeof(uint64_t) * getPreambleLongs(true) + num_entries_bytes + + wholeBytesToHoldBits(compressed_bits); +} + +template +void CompactThetaSketchAlloc::serialize(std::ostream& os) const { + const uint8_t preamble_longs = this->isEstimationMode() ? 3 + : this->isEmpty() || entries_.size() == 1 ? 1 + : 2; + write(os, preamble_longs); + write(os, UNCOMPRESSED_SERIAL_VERSION); + write(os, SKETCH_TYPE); + write(os, 0); // unused + const uint8_t flags_byte( + (1 << flags::IS_COMPACT) | (1 << flags::IS_READ_ONLY) | + (this->isEmpty() ? 1 << flags::IS_EMPTY : 0) | + (this->isOrdered() ? 1 << flags::IS_ORDERED : 0)); + write(os, flags_byte); + write(os, getSeedHash()); + if (preamble_longs > 1) { + write(os, static_cast(entries_.size())); + write(os, 0); // unused + } + if (this->isEstimationMode()) + write(os, this->theta_); + if (entries_.size() > 0) + write(os, entries_.data(), entries_.size() * sizeof(uint64_t)); +} + +template +auto CompactThetaSketchAlloc::serialize(unsigned header_size_bytes) const + -> vector_bytes { + const size_t size = getSerializedSizeBytes() + header_size_bytes; + vector_bytes bytes(size, 0, entries_.get_allocator()); + uint8_t* ptr = bytes.data() + header_size_bytes; + const uint8_t preamble_longs = getPreambleLongs(false); + *ptr++ = preamble_longs; + *ptr++ = UNCOMPRESSED_SERIAL_VERSION; + *ptr++ = SKETCH_TYPE; + ptr += sizeof(uint16_t); // unused + const uint8_t flags_byte( + (1 << flags::IS_COMPACT) | (1 << flags::IS_READ_ONLY) | + (this->isEmpty() ? 1 << flags::IS_EMPTY : 0) | + (this->isOrdered() ? 1 << flags::IS_ORDERED : 0)); + *ptr++ = flags_byte; + ptr += copyToMem(getSeedHash(), ptr); + if (preamble_longs > 1) { + ptr += copyToMem(static_cast(entries_.size()), ptr); + ptr += sizeof(uint32_t); // unused + } + if (this->isEstimationMode()) + ptr += copyToMem(theta_, ptr); + if (entries_.size() > 0) + ptr += copyToMem(entries_.data(), ptr, entries_.size() * sizeof(uint64_t)); + return bytes; +} + +template +bool CompactThetaSketchAlloc::isSuitableForCompression() const { + if (!this->isOrdered() || entries_.size() == 0 || + (entries_.size() == 1 && !this->isEstimationMode())) + return false; + return true; +} + +template +void CompactThetaSketchAlloc::serializeCompressed(std::ostream& os) const { + if (isSuitableForCompression()) + return serializeVersion4(os); + return serialize(os); +} + +template +auto CompactThetaSketchAlloc::serializeCompressed( + unsigned header_size_bytes) const -> vector_bytes { + if (isSuitableForCompression()) + return serializeVersion4(header_size_bytes); + return serialize(header_size_bytes); +} + +template +uint8_t CompactThetaSketchAlloc::computeEntryBits() const { + // compression is based on leading zeros in deltas between ordered hash values + // assumes ordered sketch + uint64_t previous = 0; + uint64_t ored = 0; + for (const uint64_t entry : entries_) { + const uint64_t delta = entry - previous; + ored |= delta; + previous = entry; + } + return 64 - countLeadingZerosInU64(ored); +} + +template +void CompactThetaSketchAlloc::serializeVersion4(std::ostream& os) const { + const uint8_t preamble_longs = this->isEstimationMode() ? 2 : 1; + const uint8_t entry_bits = computeEntryBits(); + const uint8_t num_entries_bytes = getNumEntriesBytes(); + + write(os, preamble_longs); + write(os, COMPRESSED_SERIAL_VERSION); + write(os, SKETCH_TYPE); + write(os, entry_bits); + write(os, num_entries_bytes); + const uint8_t flags_byte( + (1 << flags::IS_COMPACT) | (1 << flags::IS_READ_ONLY) | + (1 << flags::IS_ORDERED)); + write(os, flags_byte); + write(os, getSeedHash()); + if (this->isEstimationMode()) + write(os, this->theta_); + uint32_t num_entries = static_cast(entries_.size()); + for (unsigned i = 0; i < num_entries_bytes; ++i) { + write(os, num_entries & 0xff); + num_entries >>= 8; + } + + uint64_t previous = 0; + uint64_t deltas[8]; + vector_bytes buffer( + entry_bits, + 0, + entries_.get_allocator()); // block of 8 entries takes entry_bits bytes + + // pack blocks of 8 deltas + unsigned i; + for (i = 0; i + 7 < entries_.size(); i += 8) { + for (unsigned j = 0; j < 8; ++j) { + deltas[j] = entries_[i + j] - previous; + previous = entries_[i + j]; + } + packBitsBlock8(deltas, buffer.data(), entry_bits); + write(os, buffer.data(), buffer.size()); + } + + // pack extra deltas if fewer than 8 of them left + if (i < entries_.size()) { + uint8_t offset = 0; + uint8_t* ptr = buffer.data(); + for (; i < entries_.size(); ++i) { + const uint64_t delta = entries_[i] - previous; + previous = entries_[i]; + offset = packBits(delta, entry_bits, ptr, offset); + } + if (offset > 0) + ++ptr; + write(os, buffer.data(), ptr - buffer.data()); + } +} + +template +auto CompactThetaSketchAlloc::serializeVersion4( + unsigned header_size_bytes) const -> vector_bytes { + const uint8_t entry_bits = computeEntryBits(); + const uint8_t num_entries_bytes = getNumEntriesBytes(); + const size_t size = + getCompressedSerializedSizeBytes(entry_bits, num_entries_bytes) + + header_size_bytes; + vector_bytes bytes(size, 0, entries_.get_allocator()); + uint8_t* ptr = bytes.data() + header_size_bytes; + + *ptr++ = getPreambleLongs(true); + *ptr++ = COMPRESSED_SERIAL_VERSION; + *ptr++ = SKETCH_TYPE; + *ptr++ = entry_bits; + *ptr++ = num_entries_bytes; + const uint8_t flags_byte( + (1 << flags::IS_COMPACT) | (1 << flags::IS_READ_ONLY) | + (1 << flags::IS_ORDERED)); + *ptr++ = flags_byte; + ptr += copyToMem(getSeedHash(), ptr); + if (this->isEstimationMode()) { + ptr += copyToMem(theta_, ptr); + } + uint32_t num_entries = static_cast(entries_.size()); + for (unsigned i = 0; i < num_entries_bytes; ++i) { + *ptr++ = num_entries & 0xff; + num_entries >>= 8; + } + + uint64_t previous = 0; + uint64_t deltas[8]; + + // pack blocks of 8 deltas + unsigned i; + for (i = 0; i + 7 < entries_.size(); i += 8) { + for (unsigned j = 0; j < 8; ++j) { + deltas[j] = entries_[i + j] - previous; + previous = entries_[i + j]; + } + packBitsBlock8(deltas, ptr, entry_bits); + ptr += entry_bits; + } + + // pack extra deltas if fewer than 8 of them left + uint8_t offset = 0; + for (; i < entries_.size(); ++i) { + const uint64_t delta = entries_[i] - previous; + previous = entries_[i]; + offset = packBits(delta, entry_bits, ptr, offset); + } + return bytes; +} + +template +CompactThetaSketchAlloc CompactThetaSketchAlloc::deserialize( + std::istream& is, + uint64_t seed, + const A& allocator) { + const auto preamble_longs = read(is); + const auto serial_version = read(is); + const auto type = read(is); + checker::checkSketchType(type, SKETCH_TYPE); + switch (serial_version) { + case 4: + return deserializeV4(preamble_longs, is, seed, allocator); + case 3: + return deserializeV3(preamble_longs, is, seed, allocator); + case 1: + return deserializeV1(preamble_longs, is, seed, allocator); + case 2: + return deserializeV2(preamble_longs, is, seed, allocator); + default: + throw VeloxUserError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "unexpected sketch serialization version " + + std::to_string(serial_version), + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } +} + +template +CompactThetaSketchAlloc CompactThetaSketchAlloc::deserializeV1( + uint8_t, + std::istream& is, + uint64_t seed, + const A& allocator) { + const auto seed_hash = compute_seed_hash(seed); + read(is); // unused + read(is); // unused + const auto num_entries = read(is); + read(is); // unused + const auto theta = read(is); + std::vector entries(num_entries, 0, allocator); + bool isEmpty = (num_entries == 0) && (theta == ThetaConstants::MAX_THETA); + if (!isEmpty) + read(is, entries.data(), sizeof(uint64_t) * entries.size()); + if (!is.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error reading from std::istream", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + return CompactThetaSketchAlloc( + isEmpty, true, seed_hash, theta, std::move(entries)); +} + +template +CompactThetaSketchAlloc CompactThetaSketchAlloc::deserializeV2( + uint8_t preamble_longs, + std::istream& is, + uint64_t seed, + const A& allocator) { + read(is); // unused + read(is); // unused + const uint16_t seed_hash = read(is); + checker::checkSeedHash(seed_hash, compute_seed_hash(seed)); + if (preamble_longs == 1) { + if (!is.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error reading from std::istream", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + std::vector entries(0, 0, allocator); + return CompactThetaSketchAlloc( + true, true, seed_hash, ThetaConstants::MAX_THETA, std::move(entries)); + } else if (preamble_longs == 2) { + const uint32_t num_entries = read(is); + read(is); // unused + std::vector entries(num_entries, 0, allocator); + if (num_entries == 0) { + return CompactThetaSketchAlloc( + true, true, seed_hash, ThetaConstants::MAX_THETA, std::move(entries)); + } + read(is, entries.data(), entries.size() * sizeof(uint64_t)); + if (!is.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error reading from std::istream", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + return CompactThetaSketchAlloc( + false, true, seed_hash, ThetaConstants::MAX_THETA, std::move(entries)); + } else if (preamble_longs == 3) { + const uint32_t num_entries = read(is); + read(is); // unused + const auto theta = read(is); + bool isEmpty = (num_entries == 0) && (theta == ThetaConstants::MAX_THETA); + std::vector entries(num_entries, 0, allocator); + if (isEmpty) { + if (!is.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error reading from std::istream", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + return CompactThetaSketchAlloc( + true, true, seed_hash, theta, std::move(entries)); + } else { + read(is, entries.data(), sizeof(uint64_t) * entries.size()); + if (!is.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error reading from std::istream", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + return CompactThetaSketchAlloc( + false, true, seed_hash, theta, std::move(entries)); + } + } else { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + std::to_string(preamble_longs) + + " longs of premable, but expected 1, 2, or 3", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } +} + +template +CompactThetaSketchAlloc CompactThetaSketchAlloc::deserializeV3( + uint8_t preamble_longs, + std::istream& is, + uint64_t seed, + const A& allocator) { + read(is); // unused + const auto flags_byte = read(is); + const auto seed_hash = read(is); + const bool isEmpty = flags_byte & (1 << flags::IS_EMPTY); + if (!isEmpty) + checker::checkSeedHash(seed_hash, compute_seed_hash(seed)); + uint64_t theta = ThetaConstants::MAX_THETA; + uint32_t num_entries = 0; + if (!isEmpty) { + if (preamble_longs == 1) { + num_entries = 1; + } else { + num_entries = read(is); + read(is); // unused + if (preamble_longs > 2) + theta = read(is); + } + } + std::vector entries(num_entries, 0, allocator); + if (!isEmpty) + read(is, entries.data(), sizeof(uint64_t) * entries.size()); + const bool isOrdered = flags_byte & (1 << flags::IS_ORDERED); + if (!is.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error reading from std::istream", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + return CompactThetaSketchAlloc( + isEmpty, isOrdered, seed_hash, theta, std::move(entries)); +} + +template +CompactThetaSketchAlloc CompactThetaSketchAlloc::deserializeV4( + uint8_t preamble_longs, + std::istream& is, + uint64_t seed, + const A& allocator) { + const auto entry_bits = read(is); + const auto num_entries_bytes = read(is); + const auto flags_byte = read(is); + const auto seed_hash = read(is); + const bool isEmpty = flags_byte & (1 << flags::IS_EMPTY); + if (!isEmpty) + checker::checkSeedHash(seed_hash, compute_seed_hash(seed)); + uint64_t theta = ThetaConstants::MAX_THETA; + if (preamble_longs > 1) + theta = read(is); + uint32_t num_entries = 0; + for (unsigned i = 0; i < num_entries_bytes; ++i) { + num_entries |= read(is) << (i << 3); + } + vector_bytes buffer( + entry_bits, 0, allocator); // block of 8 entries takes entry_bits bytes + std::vector entries(num_entries, 0, allocator); + + // unpack blocks of 8 deltas + unsigned i; + for (i = 0; i + 7 < num_entries; i += 8) { + read(is, buffer.data(), buffer.size()); + unpackBitsBlock8(&entries[i], buffer.data(), entry_bits); + } + // unpack extra deltas if fewer than 8 of them left + if (i < num_entries) + read( + is, + buffer.data(), + wholeBytesToHoldBits((num_entries - i) * entry_bits)); + if (!is.good()) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "error reading from std::istream", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + const uint8_t* ptr = buffer.data(); + uint8_t offset = 0; + for (; i < num_entries; ++i) { + offset = unpackBits(entries[i], entry_bits, ptr, offset); + } + // undo deltas + uint64_t previous = 0; + for (i = 0; i < num_entries; ++i) { + entries[i] += previous; + previous = entries[i]; + } + const bool isOrdered = flags_byte & (1 << flags::IS_ORDERED); + return CompactThetaSketchAlloc( + isEmpty, isOrdered, seed_hash, theta, std::move(entries)); +} + +template +CompactThetaSketchAlloc CompactThetaSketchAlloc::deserialize( + const void* bytes, + size_t size, + uint64_t seed, + const A& allocator) { + auto data = CompactThetaSketchParser::parse(bytes, size, seed, false); + if (data.entryBits == 64) { // versions 1 to 3 + const uint64_t* entries = + reinterpret_cast(data.entriesStartPtr); + return CompactThetaSketchAlloc( + data.isEmpty, + data.isOrdered, + data.seedHash, + data.theta, + std::vector( + entries, entries + data.numEntries, allocator)); + } else { // version 4 + std::vector entries(data.numEntries, 0, allocator); + const uint8_t* ptr = reinterpret_cast(data.entriesStartPtr); + // unpack blocks of 8 deltas + unsigned i; + for (i = 0; i + 7 < data.numEntries; i += 8) { + unpackBitsBlock8(&entries[i], ptr, data.entryBits); + ptr += data.entryBits; + } + // unpack extra deltas if fewer than 8 of them left + uint8_t offset = 0; + for (; i < data.numEntries; ++i) { + offset = unpackBits(entries[i], data.entryBits, ptr, offset); + } + // undo deltas + uint64_t previous = 0; + for (i = 0; i < data.numEntries; ++i) { + entries[i] += previous; + previous = entries[i]; + } + return CompactThetaSketchAlloc( + data.isEmpty, + data.isOrdered, + data.seedHash, + data.theta, + std::move(entries)); + } +} + +// wrapped compact sketch + +template +WrappedCompactThetaSketchAlloc::WrappedCompactThetaSketchAlloc( + const data_type& data) + : data_(data) {} + +template +const WrappedCompactThetaSketchAlloc WrappedCompactThetaSketchAlloc::wrap( + const void* bytes, + size_t size, + uint64_t seed, + bool dump_on_error) { + return WrappedCompactThetaSketchAlloc( + CompactThetaSketchParser::parse(bytes, size, seed, dump_on_error)); +} + +template +A WrappedCompactThetaSketchAlloc::getAllocator() const { + return A(); +} + +template +bool WrappedCompactThetaSketchAlloc::isEmpty() const { + return data_.isEmpty; +} + +template +bool WrappedCompactThetaSketchAlloc::isOrdered() const { + return data_.isOrdered; +} + +template +uint64_t WrappedCompactThetaSketchAlloc::getTheta64() const { + return data_.theta; +} + +template +uint32_t WrappedCompactThetaSketchAlloc::getNumRetained() const { + return data_.numEntries; +} + +template +uint16_t WrappedCompactThetaSketchAlloc::getSeedHash() const { + return data_.seedHash; +} + +template +auto WrappedCompactThetaSketchAlloc::begin() const -> const_iterator { + return const_iterator( + data_.entriesStartPtr, data_.entryBits, data_.numEntries, 0); +} + +template +auto WrappedCompactThetaSketchAlloc::end() const -> const_iterator { + return const_iterator( + data_.entriesStartPtr, + data_.entryBits, + data_.numEntries, + data_.numEntries); +} + +template +void WrappedCompactThetaSketchAlloc::printSpecifics( + std::ostringstream&) const {} + +template +void WrappedCompactThetaSketchAlloc::printItems( + std::ostringstream& os) const { + os << "### Retained entries" << std::endl; + for (const auto hash : *this) { + os << hash << std::endl; + } + os << "### End retained entries" << std::endl; +} + +// assumes index == 0 or index == num_entries +template +WrappedCompactThetaSketchAlloc::const_iterator::const_iterator( + const void* ptr, + uint8_t entry_bits, + uint32_t num_entries, + uint32_t index) + : ptr_(ptr), + entry_bits_(entry_bits), + num_entries_(num_entries), + index_(index), + previous_(0), + is_block_mode_(num_entries_ >= 8), + offset_(0) { + if (entry_bits == 64) { // no compression + ptr_ = reinterpret_cast(ptr) + index; + } else if (index < num_entries) { + if (is_block_mode_) { + unpack8(); + } else { + unpack1(); + } + } +} + +template +auto WrappedCompactThetaSketchAlloc::const_iterator::operator++() + -> const_iterator& { + if (entry_bits_ == 64) { // no compression + ptr_ = reinterpret_cast(ptr_) + 1; + return *this; + } + if (++index_ < num_entries_) { + if (is_block_mode_) { + if ((index_ & 7) == 0) { + if (num_entries_ - index_ >= 8) { + unpack8(); + } else { + is_block_mode_ = false; + unpack1(); + } + } + } else { + unpack1(); + } + } + return *this; +} + +template +void WrappedCompactThetaSketchAlloc::const_iterator::unpack1() { + const uint32_t i = index_ & 7; + offset_ = unpackBits( + buffer_[i], + entry_bits_, + reinterpret_cast(ptr_), + offset_); + buffer_[i] += previous_; + previous_ = buffer_[i]; +} + +template +void WrappedCompactThetaSketchAlloc::const_iterator::unpack8() { + unpackBitsBlock8( + buffer_, reinterpret_cast(ptr_), entry_bits_); + ptr_ = reinterpret_cast(ptr_) + entry_bits_; + for (int i = 0; i < 8; ++i) { + buffer_[i] += previous_; + previous_ = buffer_[i]; + } +} + +template +auto WrappedCompactThetaSketchAlloc::const_iterator::operator++(int) + -> const_iterator { + const_iterator tmp(*this); + operator++(); + return tmp; +} + +template +bool WrappedCompactThetaSketchAlloc::const_iterator::operator!=( + const const_iterator& other) const { + if (entry_bits_ == 64) + return ptr_ != other.ptr_; + return index_ != other.index_; +} + +template +bool WrappedCompactThetaSketchAlloc::const_iterator::operator==( + const const_iterator& other) const { + if (entry_bits_ == 64) + return ptr_ == other.ptr_; + return index_ == other.index_; +} + +template +auto WrappedCompactThetaSketchAlloc::const_iterator::operator*() + const -> reference { + if (entry_bits_ == 64) + return *reinterpret_cast(ptr_); + return buffer_[index_ & 7]; +} + +template +auto WrappedCompactThetaSketchAlloc::const_iterator::operator->() + const -> pointer { + if (entry_bits_ == 64) + return reinterpret_cast(ptr_); + return buffer_ + (index_ & 7); +} + +} // namespace facebook::velox::common::theta + +#endif diff --git a/velox/external/theta/ThetaSketch.h b/velox/external/theta/ThetaSketch.h new file mode 100644 index 00000000000..ff31a2c064e --- /dev/null +++ b/velox/external/theta/ThetaSketch.h @@ -0,0 +1,699 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include "CompactThetaSketchParser.h" +#include "ThetaUpdateSketchBase.h" + +namespace facebook::velox::common::theta { + +// forward declarations +template +class ThetaSketchAlloc; +template +class UpdateThetaSketchAlloc; +template +class CompactThetaSketchAlloc; +template +class WrappedCompactThetaSketchAlloc; + +/// Theta sketch alias with default allocator +using thetaSketch = ThetaSketchAlloc>; +/// Update Theta sketch alias with default allocator +using updateThetaSketch = UpdateThetaSketchAlloc>; +/// Compact Theta sketch alias with default allocator +using compactThetaSketch = CompactThetaSketchAlloc>; +/// Wrapped Compact Theta sketch alias with default allocator +using wrappedCompactThetaSketch = + WrappedCompactThetaSketchAlloc>; + +/// Abstract base class for Theta sketch +template > +class BaseThetaSketchAlloc { + public: + virtual ~BaseThetaSketchAlloc() = default; + + /** + * @return allocator + */ + virtual Allocator getAllocator() const = 0; + + /** + * @return true if this sketch represents an empty set (not the same as no + * retained entries!) + */ + virtual bool isEmpty() const = 0; + + /** + * @return estimate of the distinct count of the input stream + */ + double getEstimate() const; + + /** + * Returns the approximate lower error bound given a number of standard + * deviations. This parameter is similar to the number of standard deviations + * of the normal distribution and corresponds to approximately 67%, 95% and + * 99% confidence intervals. + * @param num_std_devs number of Standard Deviations (1, 2 or 3) + * @return the lower bound + */ + double getLowerBound(uint8_t num_std_devs) const; + + /** + * Returns the approximate upper error bound given a number of standard + * deviations. This parameter is similar to the number of standard deviations + * of the normal distribution and corresponds to approximately 67%, 95% and + * 99% confidence intervals. + * @param num_std_devs number of Standard Deviations (1, 2 or 3) + * @return the upper bound + */ + double getUpperBound(uint8_t num_std_devs) const; + + /** + * @return true if the sketch is in estimation mode (as opposed to exact mode) + */ + bool isEstimationMode() const; + + /** + * @return theta as a fraction from 0 to 1 (effective sampling rate) + */ + double getTheta() const; + + /** + * @return theta as a positive integer between 0 and LLONG_MAX + */ + virtual uint64_t getTheta64() const = 0; + + /** + * @return the number of retained entries in the sketch + */ + virtual uint32_t getNumRetained() const = 0; + + /** + * @return hash of the seed that was used to hash the input + */ + virtual uint16_t getSeedHash() const = 0; + + /** + * @return true if retained entries are ordered + */ + virtual bool isOrdered() const = 0; + + /** + * Provides a human-readable summary of this sketch as a string + * @param print_items if true include the list of items retained by the sketch + * @return sketch summary as a string + */ + virtual string toString(bool print_items = false) const; + + protected: + virtual void printSpecifics(std::ostringstream& os) const = 0; + virtual void printItems(std::ostringstream& os) const = 0; +}; + +/// Base class for the Theta Sketch, a generalization of the Kth Minimum Value +/// (KMV) sketch. +template > +class ThetaSketchAlloc : public BaseThetaSketchAlloc { + public: + using Entry = uint64_t; + using ExtractKey = trivialExtractKey; + using iterator = ThetaIterator; + using const_iterator = ThetaConstIterator; + + virtual ~ThetaSketchAlloc() = default; + + /** + * Iterator over hash values in this sketch. + * @return begin iterator + */ + virtual iterator begin() = 0; + + /** + * Iterator pointing past the valid range. + * Not to be incremented or dereferenced. + * @return end iterator + */ + virtual iterator end() = 0; + + /** + * Const iterator over hash values in this sketch. + * @return begin iterator + */ + virtual const_iterator begin() const = 0; + + /** + * Const iterator pointing past the valid range. + * Not to be incremented or dereferenced. + * @return end iterator + */ + virtual const_iterator end() const = 0; + + protected: + virtual void printItems(std::ostringstream& os) const; +}; + +// forward declaration +template +class CompactThetaSketchAlloc; + +/** + * Update Theta sketch. + * The purpose of this class is to build a Theta sketch from input data via the + * update() methods. There is no constructor. Use builder instead. + */ +template > +class UpdateThetaSketchAlloc : public ThetaSketchAlloc { + public: + using Base = ThetaSketchAlloc; + using Entry = typename Base::Entry; + using ExtractKey = typename Base::ExtractKey; + using iterator = typename Base::iterator; + using const_iterator = typename Base::const_iterator; + using ThetaTable = ThetaUpdateSketchBase; + using resizeFactor = typename ThetaTable::resizeFactor; + + // No constructor here. Use builder instead. + class builder; + + /** + * Copy constructor + * @param other sketch to be copied + */ + UpdateThetaSketchAlloc(const UpdateThetaSketchAlloc& other) = default; + + /** + * Move constructor + * @param other sketch to be moved + */ + UpdateThetaSketchAlloc(UpdateThetaSketchAlloc&& other) noexcept = default; + + virtual ~UpdateThetaSketchAlloc() = default; + + /** + * Copy assignment + * @param other sketch to be copied + * @return reference to this sketch + */ + UpdateThetaSketchAlloc& operator=(const UpdateThetaSketchAlloc& other) = + default; + + /** + * Move assignment + * @param other sketch to be moved + * @return reference to this sketch + */ + UpdateThetaSketchAlloc& operator=(UpdateThetaSketchAlloc&& other) = default; + + virtual Allocator getAllocator() const override; + virtual bool isEmpty() const override; + virtual bool isOrdered() const override; + virtual uint16_t getSeedHash() const override; + virtual uint64_t getTheta64() const override; + virtual uint32_t getNumRetained() const override; + + /** + * @return configured nominal number of entries in the sketch + */ + uint8_t getLgK() const; + + /** + * @return configured resize factor of the sketch + */ + resizeFactor getRf() const; + + /** + * Update this sketch with a given string. + * @param value string to update the sketch with + */ + void update(const std::string& value); + + /** + * Update this sketch with a given unsigned 64-bit integer. + * @param value uint64_t to update the sketch with + */ + void update(uint64_t value); + + /** + * Update this sketch with a given signed 64-bit integer. + * @param value int64_t to update the sketch with + */ + void update(int64_t value); + + /** + * Update this sketch with a given unsigned 32-bit integer. + * For compatibility with Java implementation. + * @param value uint32_t to update the sketch with + */ + void update(uint32_t value); + + /** + * Update this sketch with a given signed 32-bit integer. + * For compatibility with Java implementation. + * @param value int32_t to update the sketch with + */ + void update(int32_t value); + + /** + * Update this sketch with a given unsigned 16-bit integer. + * For compatibility with Java implementation. + * @param value uint16_t to update the sketch with + */ + void update(uint16_t value); + + /** + * Update this sketch with a given signed 16-bit integer. + * For compatibility with Java implementation. + * @param value int16_t to update the sketch with + */ + void update(int16_t value); + + /** + * Update this sketch with a given unsigned 8-bit integer. + * For compatibility with Java implementation. + * @param value uint8_t to update the sketch with + */ + void update(uint8_t value); + + /** + * Update this sketch with a given signed 8-bit integer. + * For compatibility with Java implementation. + * @param value int8_t to update the sketch with + */ + void update(int8_t value); + + /** + * Update this sketch with a given double-precision floating point value. + * For compatibility with Java implementation. + * @param value double to update the sketch with + */ + void update(double value); + + /** + * Update this sketch with a given floating point value. + * For compatibility with Java implementation. + * @param value float to update the sketch with + */ + void update(float value); + + /** + * Update this sketch with given data of any type. + * This is a "universal" update that covers all cases above, + * but may produce different hashes. + * Be very careful to hash input values consistently using the same approach + * both over time and on different platforms + * and while passing sketches between C++ environment and Java environment. + * Otherwise two sketches that should represent overlapping sets will be + * disjoint For instance, for signed 32-bit values call update(int32_t) method + * above, which does widening conversion to int64_t, if compatibility with + * Java is expected + * @param data pointer to the data + * @param length of the data in bytes + */ + void update(const void* data, size_t length); + + /** + * Remove retained entries in excess of the nominal size k (if any) + */ + void trim(); + + /** + * Reset the sketch to the initial empty state + */ + void reset(); + + /** + * Converts this sketch to a compact sketch (ordered or unordered). + * @param ordered optional flag to specify if an ordered sketch should be + * produced + * @return compact sketch + */ + CompactThetaSketchAlloc compact(bool ordered = true) const; + + virtual iterator begin() override; + virtual iterator end() override; + virtual const_iterator begin() const override; + virtual const_iterator end() const override; + + private: + ThetaTable table_; + + // for builder + UpdateThetaSketchAlloc( + uint8_t lg_cur_size, + uint8_t lg_nom_size, + resizeFactor rf, + float p, + uint64_t theta, + uint64_t seed, + const Allocator& allocator); + + virtual void printSpecifics(std::ostringstream& os) const override; +}; + +/** + * Compact Theta sketch. + * This is an immutable form of the Theta sketch, the form that can be + * serialized and deserialized. + */ +template > +class CompactThetaSketchAlloc : public ThetaSketchAlloc { + public: + using Base = ThetaSketchAlloc; + using iterator = typename Base::iterator; + using const_iterator = typename Base::const_iterator; + using AllocBytes = + typename std::allocator_traits::template rebind_alloc; + using vector_bytes = std::vector; + + static const uint8_t UNCOMPRESSED_SERIAL_VERSION = 3; + static const uint8_t COMPRESSED_SERIAL_VERSION = 4; + static const uint8_t SKETCH_TYPE = 3; + + // Instances of this type can be obtained: + // - by compacting an update_theta_sketch_alloc + // - as a result of a set operation + // - by deserializing a previously serialized compact sketch + + /** + * Copy constructor. + * Constructs a compact sketch from any other type of Theta sketch + * @param other sketch to be constructed from + * @param ordered if true make the resulting sketch ordered + */ + template + CompactThetaSketchAlloc(const Other& other, bool ordered); + + /** + * Copy constructor + * @param other sketch to be copied + */ + CompactThetaSketchAlloc(const CompactThetaSketchAlloc& other) = default; + + /** + * Move constructor + * @param other sketch to be moved + */ + CompactThetaSketchAlloc(CompactThetaSketchAlloc&& other) noexcept = default; + + virtual ~CompactThetaSketchAlloc() = default; + + /** + * Copy assignment + * @param other sketch to be copied + * @return reference to this sketch + */ + CompactThetaSketchAlloc& operator=(const CompactThetaSketchAlloc& other) = + default; + + /** + * Move assignment + * @param other sketch to be moved + * @return reference to this sketch + */ + CompactThetaSketchAlloc& operator=(CompactThetaSketchAlloc&& other) = default; + + virtual Allocator getAllocator() const override; + virtual bool isEmpty() const override; + virtual bool isOrdered() const override; + virtual uint64_t getTheta64() const override; + virtual uint32_t getNumRetained() const override; + virtual uint16_t getSeedHash() const override; + + /** + * Computes maximum serialized size in bytes + * @param lg_k nominal number of entries in the sketch + */ + static size_t getMaxSerializedSizeBytes(uint8_t lg_k); + + /** + * Computes size in bytes required to serialize the current state of the + * sketch. Computing compressed size is expensive. It takes iterating over all + * retained hashes, and the actual serialization will have to look at them + * again. + * @param compressed if true compressed size is returned (if applicable) + */ + size_t getSerializedSizeBytes(bool compressed = false) const; + + /** + * This method serializes the sketch into a given stream in a binary form + * @param os output stream + */ + void serialize(std::ostream& os) const; + + /** + * This method serializes the sketch as a vector of bytes. + * An optional header can be reserved in front of the sketch. + * It is an uninitialized space of a given size. + * This header is used in Datasketches PostgreSQL extension. + * @param header_size_bytes space to reserve in front of the sketch + */ + vector_bytes serialize(unsigned header_size_bytes = 0) const; + + /** + * This method serializes the sketch into a given stream in a compressed + * binary form. Compression is applied to ordered sketches except empty and + * single item. For unordered, empty and single item sketches this method is + * equivalent to serialize() + * @param os output stream + */ + void serializeCompressed(std::ostream& os) const; + + /** + * This method serializes the sketch as a vector of bytes. + * An optional header can be reserved in front of the sketch. + * It is an uninitialized space of a given size. + * This header is used in Datasketches PostgreSQL extension. + * Compression is applied to ordered sketches except empty and single item. + * For unordered, empty and single item sketches this method is equivalent to + * serialize() + * @param header_size_bytes space to reserve in front of the sketch + */ + vector_bytes serializeCompressed(unsigned header_size_bytes = 0) const; + + virtual iterator begin() override; + virtual iterator end() override; + virtual const_iterator begin() const override; + virtual const_iterator end() const override; + + /** + * This method deserializes a sketch from a given stream. + * @param is input stream + * @param seed the seed for the hash function that was used to create the + * sketch + * @param allocator instance of an Allocator + * @return an instance of the sketch + */ + static CompactThetaSketchAlloc deserialize( + std::istream& is, + uint64_t seed = DEFAULT_SEED, + const Allocator& allocator = Allocator()); + + /** + * This method deserializes a sketch from a given array of bytes. + * @param bytes pointer to the array of bytes + * @param size the size of the array + * @param seed the seed for the hash function that was used to create the + * sketch + * @param allocator instance of an Allocator + * @return an instance of the sketch + */ + static CompactThetaSketchAlloc deserialize( + const void* bytes, + size_t size, + uint64_t seed = DEFAULT_SEED, + const Allocator& allocator = Allocator()); + + private: + enum flags { IS_BIG_ENDIAN, IS_READ_ONLY, IS_EMPTY, IS_COMPACT, IS_ORDERED }; + + bool isEmpty_; + bool isOrdered_; + uint16_t seedHash_; + uint64_t theta_; + std::vector entries_; + + uint8_t getPreambleLongs(bool compressed) const; + bool isSuitableForCompression() const; + uint8_t computeEntryBits() const; + uint8_t getNumEntriesBytes() const; + size_t getCompressedSerializedSizeBytes( + uint8_t entry_bits, + uint8_t num_entries_bytes) const; + void serializeVersion4(std::ostream& os) const; + vector_bytes serializeVersion4(unsigned header_size_bytes = 0) const; + + static CompactThetaSketchAlloc deserializeV1( + uint8_t preamble_longs, + std::istream& is, + uint64_t seed, + const Allocator& allocator); + static CompactThetaSketchAlloc deserializeV2( + uint8_t preamble_longs, + std::istream& is, + uint64_t seed, + const Allocator& allocator); + static CompactThetaSketchAlloc deserializeV3( + uint8_t preamble_longs, + std::istream& is, + uint64_t seed, + const Allocator& allocator); + static CompactThetaSketchAlloc deserializeV4( + uint8_t preamble_longs, + std::istream& is, + uint64_t seed, + const Allocator& allocator); + + virtual void printSpecifics(std::ostringstream& os) const override; + + template < + typename E, + typename EK, + typename P, + typename S, + typename CS, + typename A> + friend class ThetaUnionBase; + template < + typename E, + typename EK, + typename P, + typename S, + typename CS, + typename A> + friend class ThetaIntersectionBase; + template + friend class ThetaSetDifferenceBase; + CompactThetaSketchAlloc( + bool is_empty, + bool is_ordered, + uint16_t seed_hash, + uint64_t theta, + std::vector&& entries); +}; + +/// Update Theta sketch builder +template +class UpdateThetaSketchAlloc::builder + : public ThetaBaseBuilder { + public: + /** + * Constructor + * @param allocator + */ + builder(const Allocator& allocator = Allocator()); + /// @return instance of Update Theta sketch + UpdateThetaSketchAlloc build() const; +}; + +/** + * Wrapped Compact Theta sketch. + * This is to wrap a buffer containing a serialized compact sketch and use it in + * a set operation avoiding some cost of deserialization. It does not take the + * ownership of the buffer. + */ +template > +class WrappedCompactThetaSketchAlloc : public BaseThetaSketchAlloc { + public: + class const_iterator; + + virtual Allocator getAllocator() const override; + virtual bool isEmpty() const override; + virtual bool isOrdered() const override; + virtual uint64_t getTheta64() const override; + virtual uint32_t getNumRetained() const override; + virtual uint16_t getSeedHash() const override; + + /** + * Const iterator over hash values in this sketch. + * @return begin iterator + */ + const_iterator begin() const; + + /** + * Const iterator pointing past the valid range. + * Not to be incremented or dereferenced. + * @return end iterator + */ + const_iterator end() const; + + /** + * This method wraps a serialized compact sketch as an array of bytes. + * @param bytes pointer to the array of bytes + * @param size the size of the array + * @param seed the seed for the hash function that was used to create the + * sketch + * @param dump_on_error if true prints hex dump of the input + * @return an instance of the sketch + */ + static const WrappedCompactThetaSketchAlloc wrap( + const void* bytes, + size_t size, + uint64_t seed = DEFAULT_SEED, + bool dump_on_error = false); + + protected: + virtual void printSpecifics(std::ostringstream& os) const override; + virtual void printItems(std::ostringstream& os) const override; + + private: + using data_type = CompactThetaSketchParser::CompactThetaSketchData; + data_type data_; + + WrappedCompactThetaSketchAlloc(const data_type& data); +}; + +template +class WrappedCompactThetaSketchAlloc::const_iterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = const uint64_t; + using difference_type = void; + using pointer = value_type*; + using reference = uint64_t; + + const_iterator( + const void* ptr, + uint8_t entry_bits, + uint32_t num_entries, + uint32_t index); + const_iterator& operator++(); + const_iterator operator++(int); + bool operator==(const const_iterator& other) const; + bool operator!=(const const_iterator& other) const; + reference operator*() const; + pointer operator->() const; + + private: + const void* ptr_; + uint8_t entry_bits_; + uint32_t num_entries_; + uint32_t index_; + uint64_t previous_; + bool is_block_mode_; + uint8_t offset_; + uint64_t buffer_[8]; + + inline void unpack1(); + inline void unpack8(); +}; + +} // namespace facebook::velox::common::theta + +#include "ThetaSketch.cpp" diff --git a/velox/external/theta/ThetaUnion.cpp b/velox/external/theta/ThetaUnion.cpp new file mode 100644 index 00000000000..d26dad88c75 --- /dev/null +++ b/velox/external/theta/ThetaUnion.cpp @@ -0,0 +1,79 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#ifndef THETA_UNION_CPP +#define THETA_UNION_CPP + +#include "ThetaUnion.h" + +namespace facebook::velox::common::theta { + +template +ThetaUnionAlloc::ThetaUnionAlloc( + uint8_t lgCurSize, + uint8_t lgNomSize, + resizeFactor rf, + float p, + uint64_t theta, + uint64_t seed, + const A& allocator) + : state_( + lgCurSize, + lgNomSize, + rf, + p, + theta, + seed, + nop_policy(), + allocator) {} + +template +template +void ThetaUnionAlloc::update(FwdSketch&& sketch) { + state_.update(std::forward(sketch)); +} + +template +auto ThetaUnionAlloc::getResult(bool ordered) const -> CompactSketch { + return state_.getResult(ordered); +} + +template +void ThetaUnionAlloc::reset() { + state_.reset(); +} + +template +ThetaUnionAlloc::builder::builder(const A& allocator) + : ThetaBaseBuilder(allocator) {} + +template +auto ThetaUnionAlloc::builder::build() const -> ThetaUnionAlloc { + return ThetaUnionAlloc( + this->startingLgSize(), + this->lg_k_, + this->rf_, + this->p_, + this->startingTheta(), + this->seed_, + this->allocator_); +} + +} // namespace facebook::velox::common::theta + +#endif diff --git a/velox/external/theta/ThetaUnion.h b/velox/external/theta/ThetaUnion.h new file mode 100644 index 00000000000..e9e3cb54e3d --- /dev/null +++ b/velox/external/theta/ThetaUnion.h @@ -0,0 +1,113 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include "Serde.h" +#include "ThetaSketch.h" +#include "ThetaUnionBase.h" + +namespace facebook::velox::common::theta { + +// forward declaration +template +class ThetaUnionAlloc; + +// alias with default allocator for convenience +using ThetaUnion = ThetaUnionAlloc>; + +/** + * Theta Union. + * Computes union of Theta sketches. There is no constructor. Use builder + * instead. + */ +template > +class ThetaUnionAlloc { + public: + using Entry = uint64_t; + using ExtractKey = trivialExtractKey; + using Sketch = ThetaSketchAlloc; + using CompactSketch = CompactThetaSketchAlloc; + using resizeFactor = ThetaConstants::resizeFactor; + + // there is no payload in Theta sketch entry + struct nop_policy { + void operator()(uint64_t internal_entry, uint64_t incoming_entry) const { + unused(internal_entry); + unused(incoming_entry); + } + }; + using State = ThetaUnionBase< + Entry, + ExtractKey, + nop_policy, + Sketch, + CompactSketch, + Allocator>; + + // No constructor here. Use builder instead. + class builder; + + /** + * Update the union with a given sketch + * @param sketch to update the union with + */ + template + void update(FwdSketch&& sketch); + + /** + * Produces a copy of the current state of the union as a compact sketch. + * @param ordered optional flag to specify if an ordered sketch should be + * produced + * @return the result of the union + */ + CompactSketch getResult(bool ordered = true) const; + + /// Reset the union to the initial empty state + void reset(); + + private: + State state_; + + // for builder + ThetaUnionAlloc( + uint8_t lg_cur_size, + uint8_t lg_nom_size, + resizeFactor rf, + float p, + uint64_t theta, + uint64_t seed, + const Allocator& allocator); +}; + +/// Theta union builder +template +class ThetaUnionAlloc::builder : public ThetaBaseBuilder { + public: + builder(const A& allocator = A()); + + /** + * Create an instance of the union with predefined parameters. + * @return an instance of the union + */ + ThetaUnionAlloc build() const; +}; + +} // namespace facebook::velox::common::theta + +#include "ThetaUnion.cpp" diff --git a/velox/external/theta/ThetaUnionBase.cpp b/velox/external/theta/ThetaUnionBase.cpp new file mode 100644 index 00000000000..4567bd46e32 --- /dev/null +++ b/velox/external/theta/ThetaUnionBase.cpp @@ -0,0 +1,168 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#ifndef THETA_UNION_BASE_CPP +#define THETA_UNION_BASE_CPP + +#include + +#include "ConditionalForward.h" +#include "ThetaUnionBase.h" +#include "velox/common/base/Exceptions.h" + +namespace facebook::velox::common::theta { + +template < + typename EN, + typename EK, + typename P, + typename S, + typename CS, + typename A> +ThetaUnionBase::ThetaUnionBase( + uint8_t lg_cur_size, + uint8_t lg_nom_size, + resizeFactor rf, + float p, + uint64_t theta, + uint64_t seed, + const P& policy, + const A& allocator) + : policy_(policy), + table_(lg_cur_size, lg_nom_size, rf, p, theta, seed, allocator), + union_theta_(table_.theta_) {} + +template < + typename EN, + typename EK, + typename P, + typename S, + typename CS, + typename A> +template +void ThetaUnionBase::update(SS&& sketch) { + if (sketch.isEmpty()) + return; + if (sketch.getSeedHash() != compute_seed_hash(table_.seed_)) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "seed hash mismatch", + error_source::kErrorSourceRuntime, + error_code::kUnknown, + false /*retriable*/); + } + table_.isEmpty_ = false; + union_theta_ = std::min(union_theta_, sketch.getTheta64()); + for (auto&& entry : sketch) { + const uint64_t hash = EK()(entry); + if (hash < union_theta_ && hash < table_.theta_) { + auto result = table_.find(hash); + if (!result.second) { + table_.insert(result.first, conditionalForward(entry)); + } else { + policy_(*result.first, conditionalForward(entry)); + } + } else { + if (sketch.isOrdered()) + break; // early stop + } + } + union_theta_ = std::min(union_theta_, table_.theta_); +} + +template < + typename EN, + typename EK, + typename P, + typename S, + typename CS, + typename A> +CS ThetaUnionBase::getResult(bool ordered) const { + std::vector entries(table_.allocator_); + if (table_.isEmpty_) + return CS( + true, + true, + compute_seed_hash(table_.seed_), + union_theta_, + std::move(entries)); + entries.reserve(table_.numEntries_); + uint64_t theta = std::min(union_theta_, table_.theta_); + const uint32_t nominal_num = 1 << table_.lgNomSize_; + if (union_theta_ >= table_.theta_) { + std::copy_if( + table_.begin(), + table_.end(), + std::back_inserter(entries), + keyNotZero()); + } else { + std::copy_if( + table_.begin(), + table_.end(), + std::back_inserter(entries), + keyNotZeroLessThan(theta)); + } + if (entries.size() > nominal_num) { + std::nth_element( + entries.begin(), + entries.begin() + nominal_num, + entries.end(), + comparator()); + theta = EK()(entries[nominal_num]); + entries.erase(entries.begin() + nominal_num, entries.end()); + entries.shrink_to_fit(); + } + if (ordered) + std::sort(entries.begin(), entries.end(), comparator()); + return CS( + table_.isEmpty_, + ordered, + compute_seed_hash(table_.seed_), + theta, + std::move(entries)); +} + +template < + typename EN, + typename EK, + typename P, + typename S, + typename CS, + typename A> +const P& ThetaUnionBase::getPolicy() const { + return policy_; +} + +template < + typename EN, + typename EK, + typename P, + typename S, + typename CS, + typename A> +void ThetaUnionBase::reset() { + table_.reset(); + union_theta_ = table_.theta_; +} + +} // namespace facebook::velox::common::theta + +#endif diff --git a/velox/external/theta/ThetaUnionBase.h b/velox/external/theta/ThetaUnionBase.h new file mode 100644 index 00000000000..1d25f060191 --- /dev/null +++ b/velox/external/theta/ThetaUnionBase.h @@ -0,0 +1,65 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include "ThetaUpdateSketchBase.h" + +namespace facebook::velox::common::theta { + +template < + typename Entry, + typename ExtractKey, + typename Policy, + typename Sketch, + typename CompactSketch, + typename Allocator> +class ThetaUnionBase { + public: + using hashTable = ThetaUpdateSketchBase; + using resizeFactor = typename hashTable::resizeFactor; + using comparator = compareByKey; + + ThetaUnionBase( + uint8_t lg_cur_size, + uint8_t lg_nom_size, + resizeFactor rf, + float p, + uint64_t theta, + uint64_t seed, + const Policy& policy, + const Allocator& allocator); + + template + void update(FwdSketch&& sketch); + + CompactSketch getResult(bool ordered = true) const; + + const Policy& getPolicy() const; + + void reset(); + + private: + Policy policy_; + hashTable table_; + uint64_t union_theta_; +}; + +} // namespace facebook::velox::common::theta + +#include "ThetaUnionBase.cpp" diff --git a/velox/external/theta/ThetaUpdateSketchBase.cpp b/velox/external/theta/ThetaUpdateSketchBase.cpp new file mode 100644 index 00000000000..591c6a084a0 --- /dev/null +++ b/velox/external/theta/ThetaUpdateSketchBase.cpp @@ -0,0 +1,530 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#ifndef THETA_UPDATE_SKETCH_BASE_CPP +#define THETA_UPDATE_SKETCH_BASE_CPP + +#include +#include +#include +#include + +#include "ThetaHelpers.h" +#include "ThetaUpdateSketchBase.h" +#include "velox/common/base/Exceptions.h" + +namespace facebook::velox::common::theta { + +template +ThetaUpdateSketchBase::ThetaUpdateSketchBase( + uint8_t lgCurSize, + uint8_t lgNomSize, + resizeFactor rf, + float p, + uint64_t theta, + uint64_t seed, + const A& allocator, + bool isEmpty) + : allocator_(allocator), + isEmpty_(isEmpty), + lgCurSize_(lgCurSize), + lgNomSize_(lgNomSize), + rf_(rf), + p_(p), + numEntries_(0), + theta_(theta), + seed_(seed), + entries_(nullptr) { + if (lgCurSize > 0) { + const size_t size = 1ULL << lgCurSize; + entries_ = allocator_.allocate(size); + for (size_t i = 0; i < size; ++i) + EK()(entries_[i]) = 0; + } +} + +template +ThetaUpdateSketchBase::ThetaUpdateSketchBase( + const ThetaUpdateSketchBase& other) + : allocator_(other.allocator_), + isEmpty_(other.isEmpty_), + lgCurSize_(other.lgCurSize_), + lgNomSize_(other.lgNomSize_), + rf_(other.rf_), + p_(other.p_), + numEntries_(other.numEntries_), + theta_(other.theta_), + seed_(other.seed_), + entries_(nullptr) { + if (other.entries_ != nullptr) { + const size_t size = 1ULL << lgCurSize_; + entries_ = allocator_.allocate(size); + for (size_t i = 0; i < size; ++i) { + if (EK()(other.entries_[i]) != 0) { + new (&entries_[i]) EN(other.entries_[i]); + } else { + EK()(entries_[i]) = 0; + } + } + } +} + +template +ThetaUpdateSketchBase::ThetaUpdateSketchBase( + ThetaUpdateSketchBase&& other) noexcept + : allocator_(std::move(other.allocator_)), + isEmpty_(other.isEmpty_), + lgCurSize_(other.lgCurSize_), + lgNomSize_(other.lgNomSize_), + rf_(other.rf_), + p_(other.p_), + numEntries_(other.numEntries_), + theta_(other.theta_), + seed_(other.seed_), + entries_(other.entries_) { + other.entries_ = nullptr; +} + +template +ThetaUpdateSketchBase::~ThetaUpdateSketchBase() { + if (entries_ != nullptr) { + const size_t size = 1ULL << lgCurSize_; + for (size_t i = 0; i < size; ++i) { + if (EK()(entries_[i]) != 0) + entries_[i].~EN(); + } + allocator_.deallocate(entries_, size); + } +} + +template +ThetaUpdateSketchBase& ThetaUpdateSketchBase::operator=( + const ThetaUpdateSketchBase& other) { + ThetaUpdateSketchBase copy(other); + std::swap(allocator_, copy.allocator_); + std::swap(isEmpty_, copy.isEmpty_); + std::swap(lgCurSize_, copy.lgCurSize_); + std::swap(lgNomSize_, copy.lgNomSize_); + std::swap(rf_, copy.rf_); + std::swap(p_, copy.p_); + std::swap(numEntries_, copy.numEntries_); + std::swap(theta_, copy.theta_); + std::swap(seed_, copy.seed_); + std::swap(entries_, copy.entries_); + return *this; +} + +template +ThetaUpdateSketchBase& ThetaUpdateSketchBase::operator=( + ThetaUpdateSketchBase&& other) { + std::swap(allocator_, other.allocator_); + std::swap(isEmpty_, other.isEmpty_); + std::swap(lgCurSize_, other.lgCurSize_); + std::swap(lgNomSize_, other.lgNomSize_); + std::swap(rf_, other.rf_); + std::swap(p_, other.p_); + std::swap(numEntries_, other.numEntries_); + std::swap(theta_, other.theta_); + std::swap(seed_, other.seed_); + std::swap(entries_, other.entries_); + return *this; +} + +template +uint64_t ThetaUpdateSketchBase::hashAndScreen( + const void* data, + size_t length) { + isEmpty_ = false; + const uint64_t hash = computeHash(data, length, seed_); + if (hash >= theta_) + return 0; // hash == 0 is reserved to mark empty slots in the table + return hash; +} + +template +auto ThetaUpdateSketchBase::find(uint64_t key) const + -> std::pair { + return find(entries_, lgCurSize_, key); +} + +template +auto ThetaUpdateSketchBase::find( + EN* entries, + uint8_t lg_size, + uint64_t key) -> std::pair { + const uint32_t size = 1 << lg_size; + const uint32_t mask = size - 1; + const uint32_t stride = getStride(key, lg_size); + uint32_t index = static_cast(key) & mask; + // search for duplicate or zero + const uint32_t loop_index = index; + do { + const uint64_t probe = EK()(entries[index]); + if (probe == 0) { + return std::pair(&entries[index], false); + } else if (probe == key) { + return std::pair(&entries[index], true); + } + index = (index + stride) & mask; + } while (index != loop_index); + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "key not found and no empty slots!", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); +} + +template +template +void ThetaUpdateSketchBase::insert(iterator it, Fwd&& entry) { + new (it) EN(std::forward(entry)); + ++numEntries_; + if (numEntries_ > getCapacity(lgCurSize_, lgNomSize_)) { + if (lgCurSize_ <= lgNomSize_) { + resize(); + } else { + rebuild(); + } + } +} + +template +auto ThetaUpdateSketchBase::begin() const -> iterator { + return entries_; +} + +template +auto ThetaUpdateSketchBase::end() const -> iterator { + return entries_ + (1ULL << lgCurSize_); +} + +template +uint32_t ThetaUpdateSketchBase::getCapacity( + uint8_t lg_cur_size, + uint8_t lg_nom_size) { + const double fraction = + (lg_cur_size <= lg_nom_size) ? RESIZE_THRESHOLD : REBUILD_THRESHOLD; + return static_cast(std::floor(fraction * (1 << lg_cur_size))); +} + +template +uint32_t ThetaUpdateSketchBase::getStride( + uint64_t key, + uint8_t lg_size) { + // odd and independent of index assuming lg_size lowest bits of the key were + // used for the index + return (2 * static_cast((key >> lg_size) & STRIDE_MASK)) + 1; +} + +template +void ThetaUpdateSketchBase::resize() { + const size_t old_size = 1ULL << lgCurSize_; + const uint8_t lg_new_size = + std::min(lgCurSize_ + static_cast(rf_), lgNomSize_ + 1); + const size_t new_size = 1ULL << lg_new_size; + EN* new_entries = allocator_.allocate(new_size); + for (size_t i = 0; i < new_size; ++i) + EK()(new_entries[i]) = 0; + for (size_t i = 0; i < old_size; ++i) { + const uint64_t key = EK()(entries_[i]); + if (key != 0) { + // always finds an empty slot in a larger table + new (find(new_entries, lg_new_size, key).first) + EN(std::move(entries_[i])); + entries_[i].~EN(); + EK()(entries_[i]) = 0; + } + } + std::swap(entries_, new_entries); + lgCurSize_ = lg_new_size; + allocator_.deallocate(new_entries, old_size); +} + +// assumes number of entries > nominal size +template +void ThetaUpdateSketchBase::rebuild() { + const size_t size = 1ULL << lgCurSize_; + const uint32_t nominal_size = 1 << lgNomSize_; + + // empty entries have uninitialized payloads + // TODO: avoid this for empty or trivial payloads (arithmetic types) + consolidateNonEmpty(entries_, size, numEntries_); + + std::nth_element( + entries_, entries_ + nominal_size, entries_ + numEntries_, comparator()); + this->theta_ = EK()(entries_[nominal_size]); + EN* old_entries = entries_; + const size_t num_old_entries = numEntries_; + entries_ = allocator_.allocate(size); + for (size_t i = 0; i < size; ++i) + EK()(entries_[i]) = 0; + numEntries_ = nominal_size; + // relies on consolidating non-empty entries to the front + for (size_t i = 0; i < nominal_size; ++i) { + new (find(EK()(old_entries[i])).first) EN(std::move(old_entries[i])); + old_entries[i].~EN(); + } + for (size_t i = nominal_size; i < num_old_entries; ++i) + old_entries[i].~EN(); + allocator_.deallocate(old_entries, size); +} + +template +void ThetaUpdateSketchBase::trim() { + if (numEntries_ > static_cast(1 << lgNomSize_)) + rebuild(); +} + +template +void ThetaUpdateSketchBase::reset() { + const size_t cur_size = 1ULL << lgCurSize_; + for (size_t i = 0; i < cur_size; ++i) { + if (EK()(entries_[i]) != 0) { + entries_[i].~EN(); + EK()(entries_[i]) = 0; + } + } + const uint8_t starting_lg_size = ThetaBuildHelper::startingSubMultiple( + lgNomSize_ + 1, ThetaConstants::MIN_LG_K, static_cast(rf_)); + if (starting_lg_size != lgCurSize_) { + allocator_.deallocate(entries_, cur_size); + lgCurSize_ = starting_lg_size; + const size_t new_size = 1ULL << starting_lg_size; + entries_ = allocator_.allocate(new_size); + for (size_t i = 0; i < new_size; ++i) + EK()(entries_[i]) = 0; + } + numEntries_ = 0; + theta_ = ThetaBuildHelper::startingThetaFromP(p_); + isEmpty_ = true; +} + +template +void ThetaUpdateSketchBase::consolidateNonEmpty( + EN* entries, + size_t size, + size_t num) { + // find the first empty slot + size_t i = 0; + while (i < size) { + if (EK()(entries[i]) == 0) + break; + ++i; + } + // scan the rest and move non-empty entries to the front + for (size_t j = i + 1; j < size; ++j) { + if (EK()(entries[j]) != 0) { + new (&entries[i]) EN(std::move(entries[j])); + entries[j].~EN(); + EK()(entries[j]) = 0; + ++i; + if (i == num) + break; + } + } +} + +// builder + +template +ThetaBaseBuilder::ThetaBaseBuilder( + const Allocator& allocator) + : allocator_(allocator), + lg_k_(ThetaConstants::DEFAULT_LG_K), + rf_(ThetaConstants::DEFAULT_RESIZE_FACTOR), + p_(1), + seed_(DEFAULT_SEED) {} + +template +Derived& ThetaBaseBuilder::set_lg_k(uint8_t lg_k) { + if (lg_k < ThetaConstants::MIN_LG_K) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "lg_k must not be less than " + + std::to_string(ThetaConstants::MIN_LG_K) + ": " + std::to_string(lg_k), + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + if (lg_k > ThetaConstants::MAX_LG_K) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "lg_k must not be greater than " + + std::to_string(ThetaConstants::MAX_LG_K) + ": " + std::to_string(lg_k), + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + lg_k_ = lg_k; + return static_cast(*this); +} + +template +Derived& ThetaBaseBuilder::setResizeFactor( + resizeFactor rf) { + rf_ = rf; + return static_cast(*this); +} + +template +Derived& ThetaBaseBuilder::setP(float p) { + if (p <= 0 || p > 1) { + throw VeloxRuntimeError( + __FILE__, + __LINE__, + __FUNCTION__, + "", + "sampling probability must be between 0 and 1", + error_source::kErrorSourceRuntime, + error_code::kInvalidArgument, + false /*retriable*/); + } + p_ = p; + return static_cast(*this); +} + +template +Derived& ThetaBaseBuilder::setSeed(uint64_t seed) { + seed_ = seed; + return static_cast(*this); +} + +template +uint64_t ThetaBaseBuilder::startingTheta() const { + return ThetaBuildHelper::startingThetaFromP(p_); +} + +template +uint8_t ThetaBaseBuilder::startingLgSize() const { + return ThetaBuildHelper::startingSubMultiple( + lg_k_ + 1, ThetaConstants::MIN_LG_K, static_cast(rf_)); +} + +// iterator + +template +ThetaIterator::ThetaIterator( + Entry* entries, + uint32_t size, + uint32_t index) + : entries_(entries), size_(size), index_(index) { + while (index_ < size_ && ExtractKey()(entries_[index_]) == 0) + ++index_; +} + +template +auto ThetaIterator::operator++() -> ThetaIterator& { + ++index_; + while (index_ < size_ && ExtractKey()(entries_[index_]) == 0) + ++index_; + return *this; +} + +template +auto ThetaIterator::operator++(int) -> ThetaIterator { + ThetaIterator tmp(*this); + operator++(); + return tmp; +} + +template +bool ThetaIterator::operator!=( + const ThetaIterator& other) const { + return index_ != other.index_; +} + +template +bool ThetaIterator::operator==( + const ThetaIterator& other) const { + return index_ == other.index_; +} + +template +auto ThetaIterator::operator*() const -> reference { + return entries_[index_]; +} + +template +auto ThetaIterator::operator->() const -> pointer { + return entries_ + index_; +} + +// const iterator + +template +ThetaConstIterator::ThetaConstIterator( + const Entry* entries, + uint32_t size, + uint32_t index) + : entries_(entries), size_(size), index_(index) { + while (index_ < size_ && ExtractKey()(entries_[index_]) == 0) + ++index_; +} + +template +auto ThetaConstIterator::operator++() + -> ThetaConstIterator& { + ++index_; + while (index_ < size_ && ExtractKey()(entries_[index_]) == 0) + ++index_; + return *this; +} + +template +auto ThetaConstIterator::operator++(int) + -> ThetaConstIterator { + ThetaConstIterator tmp(*this); + operator++(); + return tmp; +} + +template +bool ThetaConstIterator::operator!=( + const ThetaConstIterator& other) const { + return index_ != other.index_; +} + +template +bool ThetaConstIterator::operator==( + const ThetaConstIterator& other) const { + return index_ == other.index_; +} + +template +auto ThetaConstIterator::operator*() const -> reference { + return entries_[index_]; +} + +template +auto ThetaConstIterator::operator->() const -> pointer { + return entries_ + index_; +} + +} // namespace facebook::velox::common::theta + +#endif diff --git a/velox/external/theta/ThetaUpdateSketchBase.h b/velox/external/theta/ThetaUpdateSketchBase.h new file mode 100644 index 00000000000..e0c98928173 --- /dev/null +++ b/velox/external/theta/ThetaUpdateSketchBase.h @@ -0,0 +1,258 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include +#include + +#include "MurmurHash3.h" +#include "ThetaComparators.h" +#include "ThetaConstants.h" + +namespace facebook::velox::common::theta { + +template +struct ThetaUpdateSketchBase { + using resizeFactor = ThetaConstants::resizeFactor; + using comparator = compareByKey; + + ThetaUpdateSketchBase( + uint8_t lg_cur_size, + uint8_t lg_nom_size, + resizeFactor rf, + float p, + uint64_t theta, + uint64_t seed, + const Allocator& allocator, + bool is_empty = true); + ThetaUpdateSketchBase(const ThetaUpdateSketchBase& other); + ThetaUpdateSketchBase(ThetaUpdateSketchBase&& other) noexcept; + ~ThetaUpdateSketchBase(); + ThetaUpdateSketchBase& operator=(const ThetaUpdateSketchBase& other); + ThetaUpdateSketchBase& operator=(ThetaUpdateSketchBase&& other); + + using iterator = Entry*; + + inline uint64_t hashAndScreen(const void* data, size_t length); + + inline std::pair find(uint64_t key) const; + static inline std::pair + find(Entry* entries, uint8_t lg_size, uint64_t key); + + template + inline void insert(iterator it, FwdEntry&& entry); + + iterator begin() const; + iterator end() const; + + // resize threshold = 0.5 tuned for speed + static constexpr double RESIZE_THRESHOLD = 0.5; + // hash table rebuild threshold = 15/16 + static constexpr double REBUILD_THRESHOLD = 15.0 / 16.0; + + static constexpr uint8_t STRIDE_HASH_BITS = 7; + static constexpr uint32_t STRIDE_MASK = (1 << STRIDE_HASH_BITS) - 1; + + Allocator allocator_; + bool isEmpty_; + uint8_t lgCurSize_; + uint8_t lgNomSize_; + resizeFactor rf_; + float p_; + uint32_t numEntries_; + uint64_t theta_; + uint64_t seed_; + Entry* entries_; + + void resize(); + void rebuild(); + void trim(); + void reset(); + + static inline uint32_t getCapacity(uint8_t lg_cur_size, uint8_t lg_nom_size); + static inline uint32_t getStride(uint64_t key, uint8_t lg_size); + static void consolidateNonEmpty(Entry* entries, size_t size, size_t num); +}; + +/// Theta base builder +template +class ThetaBaseBuilder { + public: + /** + * Creates and instance of the builder with default parameters. + * @param allocator instance of an Allocator to pass to created sketches + */ + ThetaBaseBuilder(const Allocator& allocator); + + /** + * Set log2(k), where k is a nominal number of entries in the sketch + * @param lg_k base 2 logarithm of nominal number of entries + * @return this builder + */ + Derived& set_lg_k(uint8_t lg_k); + + /** + * Set resize factor for the internal hash table (defaults to 8) + * @param rf resize factor + * @return this builder + */ + Derived& setResizeFactor(resizeFactor rf); + + /** + * Set sampling probability (initial theta). The default is 1, so the sketch + * retains all entries until it reaches the limit, at which point it goes into + * the estimation mode and reduces the effective sampling probability (theta) + * as necessary. + * @param p sampling probability + * @return this builder + */ + Derived& setP(float p); + + /** + * Set the seed for the hash function. Should be used carefully if needed. + * Sketches produced with different seed are not compatible + * and cannot be mixed in set operations. + * @param seed hash seed + * @return this builder + */ + Derived& setSeed(uint64_t seed); + + protected: + Allocator allocator_; + uint8_t lg_k_; + resizeFactor rf_; + float p_; + uint64_t seed_; + + uint64_t startingTheta() const; + uint8_t startingLgSize() const; +}; + +// key extractor + +struct trivialExtractKey { + template + auto operator()(T&& entry) const -> decltype(std::forward(entry)) { + return std::forward(entry); + } +}; + +// key not zero + +template +class keyNotZero { + public: + bool operator()(const Entry& entry) const { + return ExtractKey()(entry) != 0; + } +}; + +template +class keyNotZeroLessThan { + public: + explicit keyNotZeroLessThan(const Key& key) : key(key) {} + bool operator()(const Entry& entry) const { + return ExtractKey()(entry) != 0 && ExtractKey()(entry) < this->key; + } + + private: + Key key; +}; + +// MurMur3 hash functions + +static inline uint64_t +computeHash(const void* data, size_t length, uint64_t seed) { + HashState hashes; + MurmurHash3_x64_128(data, length, seed, hashes); + return ( + hashes.h1 >> + 1); // Java implementation does unsigned shift >>> to make values positive +} + +// iterators + +template +class ThetaIterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = Entry; + using difference_type = std::ptrdiff_t; + using pointer = Entry*; + using reference = Entry&; + + ThetaIterator(Entry* entries, uint32_t size, uint32_t index); + ThetaIterator& operator++(); + ThetaIterator operator++(int); + bool operator==(const ThetaIterator& other) const; + bool operator!=(const ThetaIterator& other) const; + reference operator*() const; + pointer operator->() const; + + private: + Entry* entries_; + uint32_t size_; + uint32_t index_; +}; + +template +class ThetaConstIterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = const Entry; + using difference_type = std::ptrdiff_t; + using pointer = const Entry*; + using reference = const Entry&; + + ThetaConstIterator(const Entry* entries, uint32_t size, uint32_t index); + ThetaConstIterator& operator++(); + ThetaConstIterator operator++(int); + bool operator==(const ThetaConstIterator& other) const; + bool operator!=(const ThetaConstIterator& other) const; + reference operator*() const; + pointer operator->() const; + + private: + const Entry* entries_; + uint32_t size_; + uint32_t index_; +}; + +// double value canonicalization for compatibility with Java +static inline int64_t canonical_double(double value) { + union { + int64_t long_value; + double double_value; + } long_double_union; + + if (value == 0.0) { + long_double_union.double_value = 0.0; // canonicalize -0.0 to 0.0 + } else if (std::isnan(value)) { + long_double_union.long_value = + 0x7ff8000000000000L; // canonicalize NaN using value from Java's + // Double.doubleToLongBits() + } else { + long_double_union.double_value = value; + } + return long_double_union.long_value; +} + +} // namespace facebook::velox::common::theta + +#include "ThetaUpdateSketchBase.cpp" diff --git a/velox/external/theta/tests/BitPackingTest.cpp b/velox/external/theta/tests/BitPackingTest.cpp new file mode 100644 index 00000000000..0eff4caed1e --- /dev/null +++ b/velox/external/theta/tests/BitPackingTest.cpp @@ -0,0 +1,133 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +// Adapted from Apache DataSketches + +#include "velox/external/theta/BitPacking.h" + +#include + +namespace facebook::velox::common::theta { + +// for every number of bits from 1 to 63 +// generate pseudo-random data, pack, unpack and compare + +// inverse golden ratio (0.618.. of max uint64_t) +static const uint64_t IGOLDEN64 = 0x9e3779b97f4a7c13ULL; + +TEST(BitPackingTest, packUnpackBits) { + uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value + for (int m = 0; m < 10000; ++m) { + for (uint8_t bits = 1; bits <= 63; ++bits) { + int n = 8; + const uint64_t mask = (1ULL << bits) - 1; + std::vector input(n, 0); + for (int i = 0; i < n; ++i) { + input[i] = value & mask; + value += IGOLDEN64; + } + std::vector bytes(n * sizeof(uint64_t), 0); + uint8_t offset = 0; + uint8_t* ptr = bytes.data(); + for (int i = 0; i < n; ++i) { + offset = packBits(input[i], bits, ptr, offset); + } + + std::vector output(n, 0); + offset = 0; + const uint8_t* cptr = bytes.data(); + for (int i = 0; i < n; ++i) { + offset = unpackBits(output[i], bits, cptr, offset); + } + for (int i = 0; i < n; ++i) { + EXPECT_EQ(input[i], output[i]); + } + } + } +} + +TEST(BitPackingTest, packUnpackBlocks) { + uint64_t value = 0xaa55aa55aa55aa55ULL; // arbitrary starting value + for (int n = 0; n < 10000; ++n) { + for (uint8_t bits = 1; bits <= 63; ++bits) { + const uint64_t mask = (1ULL << bits) - 1; + std::vector input(8, 0); + for (int i = 0; i < 8; ++i) { + input[i] = value & mask; + value += IGOLDEN64; + } + std::vector bytes(bits, 0); + packBitsBlock8(input.data(), bytes.data(), bits); + std::vector output(8, 0); + unpackBitsBlock8(output.data(), bytes.data(), bits); + for (int i = 0; i < 8; ++i) { + EXPECT_EQ(input[i], output[i]); + } + } + } +} + +TEST(BitPackingTest, packBitsUnpackBlocks) { + uint64_t value = 0; // arbitrary starting value + for (int m = 0; m < 10000; ++m) { + for (uint8_t bits = 1; bits <= 63; ++bits) { + const uint64_t mask = (1ULL << bits) - 1; + std::vector input(8, 0); + for (int i = 0; i < 8; ++i) { + input[i] = value & mask; + value += IGOLDEN64; + } + std::vector bytes(bits, 0); + uint8_t offset = 0; + uint8_t* ptr = bytes.data(); + for (int i = 0; i < 8; ++i) { + offset = packBits(input[i], bits, ptr, offset); + } + std::vector output(8, 0); + unpackBitsBlock8(output.data(), bytes.data(), bits); + for (int i = 0; i < 8; ++i) { + EXPECT_EQ(input[i], output[i]); + } + } + } +} + +TEST(BitPackingTest, packBlocksUnpackBits) { + uint64_t value = 111; // arbitrary starting value + for (int m = 0; m < 10000; ++m) { + for (uint8_t bits = 1; bits <= 63; ++bits) { + const uint64_t mask = (1ULL << bits) - 1; + std::vector input(8, 0); + for (int i = 0; i < 8; ++i) { + input[i] = value & mask; + value += IGOLDEN64; + } + std::vector bytes(bits, 0); + packBitsBlock8(input.data(), bytes.data(), bits); + std::vector output(8, 0); + uint8_t offset = 0; + const uint8_t* cptr = bytes.data(); + for (int i = 0; i < 8; ++i) { + offset = unpackBits(output[i], bits, cptr, offset); + } + for (int i = 0; i < 8; ++i) { + EXPECT_EQ(input[i], output[i]); + } + } + } +} + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/tests/CMakeLists.txt b/velox/external/theta/tests/CMakeLists.txt new file mode 100644 index 00000000000..431e13e1ce9 --- /dev/null +++ b/velox/external/theta/tests/CMakeLists.txt @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +add_executable( + velox_common_theta_test + ThetaSketchTest.cpp + ThetaUnionTest.cpp + BitPackingTest.cpp + TestUtils.h) + +add_test(NAME velox_common_theta_test COMMAND velox_common_theta_test) + +target_link_libraries( + velox_common_theta_test + PRIVATE velox_common_theta GTest::gtest GTest::gtest_main) + +file(COPY test_sketch_files + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/velox/external/theta/tests/TestUtils.h b/velox/external/theta/tests/TestUtils.h new file mode 100644 index 00000000000..a9055b521ab --- /dev/null +++ b/velox/external/theta/tests/TestUtils.h @@ -0,0 +1,204 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#pragma once + +#include +#include +#include "velox/common/base/Exceptions.h" + +namespace { +class Approx { + private: + bool marginComparison(double lhs, double rhs, double margin) const { + return (lhs + margin >= rhs) && (rhs + margin >= lhs); + } + + bool equalityComparisonImpl(const double other) const { + // First try with fixed margin, then compute margin based on epsilon, scale + // and Approx's value Thanks to Richard Harris for his help refining the + // scaled margin value + return marginComparison(m_value, other, m_margin) || + marginComparison( + m_value, + other, + m_epsilon * + (m_scale + std::fabs(std::isinf(m_value) ? 0 : m_value))); + } + // Validates the new margin (margin >= 0) + // out-of-line to avoid including stdexcept in the header + void setMargin(double newMargin) { + VELOX_CHECK( + newMargin >= 0, + "Invalid Approx::margin: {} Approx::Margin has to be non-negative.", + newMargin); + m_margin = newMargin; + } + + // Validates the new epsilon (0 < epsilon < 1) + // out-of-line to avoid including stdexcept in the header + void setEpsilon(double newEpsilon) { + VELOX_CHECK( + newEpsilon >= 0 && newEpsilon <= 1.0, + "Invalid Approx::epsilon: {} Approx::epsilon has to be in [0, 1]", + newEpsilon); + m_epsilon = newEpsilon; + } + + public: + explicit Approx(double value) + : m_epsilon(std::numeric_limits::epsilon() * 100), + m_margin(0.0), + m_scale(0.0), + m_value(value) {} + + static Approx custom() { + return Approx(0); + } + + Approx operator-() const { + auto temp(*this); + temp.m_value = -temp.m_value; + return temp; + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + Approx operator()(T const& value) const { + Approx approx(static_cast(value)); + approx.m_epsilon = m_epsilon; + approx.m_margin = m_margin; + approx.m_scale = m_scale; + return approx; + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + explicit Approx(T const& value) : Approx(static_cast(value)) {} + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + friend bool operator==(const T& lhs, Approx const& rhs) { + auto lhs_v = static_cast(lhs); + return rhs.equalityComparisonImpl(lhs_v); + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + friend bool operator==(Approx const& lhs, const T& rhs) { + return operator==(rhs, lhs); + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + friend bool operator!=(T const& lhs, Approx const& rhs) { + return !operator==(lhs, rhs); + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + friend bool operator!=(Approx const& lhs, T const& rhs) { + return !operator==(rhs, lhs); + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + friend bool operator<=(T const& lhs, Approx const& rhs) { + return static_cast(lhs) < rhs.m_value || lhs == rhs; + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + friend bool operator<=(Approx const& lhs, T const& rhs) { + return lhs.m_value < static_cast(rhs) || lhs == rhs; + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + friend bool operator>=(T const& lhs, Approx const& rhs) { + return static_cast(lhs) > rhs.m_value || lhs == rhs; + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + friend bool operator>=(Approx const& lhs, T const& rhs) { + return lhs.m_value > static_cast(rhs) || lhs == rhs; + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + Approx& epsilon(T const& newEpsilon) { + double epsilonAsDouble = static_cast(newEpsilon); + setEpsilon(epsilonAsDouble); + return *this; + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + Approx& margin(T const& newMargin) { + double marginAsDouble = static_cast(newMargin); + setMargin(marginAsDouble); + return *this; + } + + template < + typename T, + typename = typename std::enable_if< + std::is_constructible::value>::type> + Approx& scale(T const& newScale) { + m_scale = static_cast(newScale); + return *this; + } + + std::string toString() const { + std::stringstream rss; + rss << "Approx( " << m_value << " )"; + return rss.str(); + } + + private: + double m_epsilon; + double m_margin; + double m_scale; + double m_value; +}; +} // namespace diff --git a/velox/external/theta/tests/ThetaSketchTest.cpp b/velox/external/theta/tests/ThetaSketchTest.cpp new file mode 100644 index 00000000000..fb5ee1fced2 --- /dev/null +++ b/velox/external/theta/tests/ThetaSketchTest.cpp @@ -0,0 +1,773 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#include "velox/external/theta/ThetaSketch.h" +#include "TestUtils.h" + +#include +#include +#include +#include +#include +#include + +namespace facebook::velox::common::theta { + +const std::string inputPath = "test_sketch_files/"; + +TEST(ThetaSketch, Empty) { + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + EXPECT_TRUE(update_sketch.isEmpty()); + EXPECT_FALSE(update_sketch.isEstimationMode()); + EXPECT_TRUE(update_sketch.getTheta() == 1.0); + EXPECT_TRUE(update_sketch.getEstimate() == 0.0); + EXPECT_TRUE(update_sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(update_sketch.getUpperBound(1) == 0.0); + EXPECT_TRUE(update_sketch.isOrdered()); + + compactThetaSketch compact_sketch = update_sketch.compact(); + EXPECT_TRUE(compact_sketch.isEmpty()); + EXPECT_FALSE(compact_sketch.isEstimationMode()); + EXPECT_TRUE(compact_sketch.getTheta() == 1.0); + EXPECT_TRUE(compact_sketch.getEstimate() == 0.0); + EXPECT_TRUE(compact_sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(compact_sketch.getUpperBound(1) == 0.0); + EXPECT_TRUE(compact_sketch.isOrdered()); + + // empty is forced to be ordered + EXPECT_TRUE(update_sketch.compact(false).isOrdered()); +} + +TEST(ThetaSketch, NonEmptyNoRetainedKeys) { + updateThetaSketch update_sketch = + updateThetaSketch::builder().setP(0.001f).build(); + update_sketch.update(1); + EXPECT_TRUE(update_sketch.getNumRetained() == 0); + EXPECT_FALSE(update_sketch.isEmpty()); + EXPECT_TRUE(update_sketch.isEstimationMode()); + EXPECT_TRUE(update_sketch.getEstimate() == 0.0); + EXPECT_TRUE(update_sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(update_sketch.getUpperBound(1) > 0); + + compactThetaSketch compact_sketch = update_sketch.compact(); + EXPECT_TRUE(compact_sketch.getNumRetained() == 0); + EXPECT_FALSE(compact_sketch.isEmpty()); + EXPECT_TRUE(compact_sketch.isEstimationMode()); + EXPECT_TRUE(compact_sketch.getEstimate() == 0.0); + EXPECT_TRUE(compact_sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(compact_sketch.getUpperBound(1) > 0); + + update_sketch.reset(); + EXPECT_TRUE(update_sketch.isEmpty()); + EXPECT_FALSE(update_sketch.isEstimationMode()); + EXPECT_TRUE(update_sketch.getTheta() == 1.0); + EXPECT_TRUE(update_sketch.getEstimate() == 0.0); + EXPECT_TRUE(update_sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(update_sketch.getUpperBound(1) == 0.0); +} + +TEST(ThetaSketch, SingleItem) { + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + update_sketch.update(1); + EXPECT_FALSE(update_sketch.isEmpty()); + EXPECT_FALSE(update_sketch.isEstimationMode()); + EXPECT_TRUE(update_sketch.getTheta() == 1.0); + EXPECT_TRUE(update_sketch.getEstimate() == 1.0); + EXPECT_TRUE(update_sketch.getLowerBound(1) == 1.0); + EXPECT_TRUE(update_sketch.getUpperBound(1) == 1.0); + EXPECT_TRUE(update_sketch.isOrdered()); // one item is ordered + + compactThetaSketch compact_sketch = update_sketch.compact(); + EXPECT_FALSE(compact_sketch.isEmpty()); + EXPECT_FALSE(compact_sketch.isEstimationMode()); + EXPECT_TRUE(compact_sketch.getTheta() == 1.0); + EXPECT_TRUE(compact_sketch.getEstimate() == 1.0); + EXPECT_TRUE(compact_sketch.getLowerBound(1) == 1.0); + EXPECT_TRUE(compact_sketch.getUpperBound(1) == 1.0); + EXPECT_TRUE(compact_sketch.isOrdered()); + + // single item is forced to be ordered + EXPECT_TRUE(update_sketch.compact(false).isOrdered()); +} + +TEST(ThetaSketch, ResizeExact) { + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + for (int i = 0; i < 2000; i++) + update_sketch.update(i); + EXPECT_FALSE(update_sketch.isEmpty()); + EXPECT_FALSE(update_sketch.isEstimationMode()); + EXPECT_TRUE(update_sketch.getTheta() == 1.0); + EXPECT_TRUE(update_sketch.getEstimate() == 2000.0); + EXPECT_TRUE(update_sketch.getLowerBound(1) == 2000.0); + EXPECT_TRUE(update_sketch.getUpperBound(1) == 2000.0); + EXPECT_FALSE(update_sketch.isOrdered()); + + compactThetaSketch compact_sketch = update_sketch.compact(); + EXPECT_FALSE(compact_sketch.isEmpty()); + EXPECT_FALSE(compact_sketch.isEstimationMode()); + EXPECT_TRUE(compact_sketch.getTheta() == 1.0); + EXPECT_TRUE(compact_sketch.getEstimate() == 2000.0); + EXPECT_TRUE(compact_sketch.getLowerBound(1) == 2000.0); + EXPECT_TRUE(compact_sketch.getUpperBound(1) == 2000.0); + EXPECT_TRUE(compact_sketch.isOrdered()); + + update_sketch.reset(); + EXPECT_TRUE(update_sketch.isEmpty()); + EXPECT_FALSE(update_sketch.isEstimationMode()); + EXPECT_TRUE(update_sketch.getTheta() == 1.0); + EXPECT_TRUE(update_sketch.getEstimate() == 0.0); + EXPECT_TRUE(update_sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(update_sketch.getUpperBound(1) == 0.0); + EXPECT_TRUE(update_sketch.isOrdered()); +} + +TEST(ThetaSketch, estimation) { + updateThetaSketch update_sketch = + updateThetaSketch::builder() + .setResizeFactor(updateThetaSketch::resizeFactor::X1) + .build(); + const int n = 8000; + for (int i = 0; i < n; i++) + update_sketch.update(i); + // std::cerr << update_sketch.to_string(); + EXPECT_FALSE(update_sketch.isEmpty()); + EXPECT_TRUE(update_sketch.isEstimationMode()); + EXPECT_TRUE(update_sketch.getTheta() < 1.0); + EXPECT_TRUE( + update_sketch.getEstimate() == Approx((double)n).margin(n * 0.01)); + EXPECT_TRUE(update_sketch.getLowerBound(1) < n); + EXPECT_TRUE(update_sketch.getUpperBound(1) > n); + + const uint32_t k = 1 << ThetaConstants::DEFAULT_LG_K; + EXPECT_TRUE(update_sketch.getNumRetained() >= k); + update_sketch.trim(); + EXPECT_TRUE(update_sketch.getNumRetained() == k); + + compactThetaSketch compact_sketch = update_sketch.compact(); + EXPECT_FALSE(compact_sketch.isEmpty()); + EXPECT_TRUE(compact_sketch.isOrdered()); + EXPECT_TRUE(compact_sketch.isEstimationMode()); + EXPECT_TRUE(compact_sketch.getTheta() < 1.0); + EXPECT_TRUE( + compact_sketch.getEstimate() == Approx((double)n).margin(n * 0.01)); + EXPECT_TRUE(compact_sketch.getLowerBound(1) < n); + EXPECT_TRUE(compact_sketch.getUpperBound(1) > n); +} + +TEST(ThetaSketch, DeserializeCompactV1EmptyFromJava) { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(inputPath + "theta_compact_empty_from_java_v1.sk", std::ios::binary); + auto sketch = compactThetaSketch::deserialize(is); + EXPECT_TRUE(sketch.isEmpty()); + EXPECT_FALSE(sketch.isEstimationMode()); + EXPECT_TRUE(sketch.getNumRetained() == 0); + EXPECT_TRUE(sketch.getTheta() == 1.0); + EXPECT_TRUE(sketch.getEstimate() == 0.0); + EXPECT_TRUE(sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(sketch.getUpperBound(1) == 0.0); +} + +TEST(ThetaSketch, DeserializeCompactV2EmptyFromJava) { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open(inputPath + "theta_compact_empty_from_java_v2.sk", std::ios::binary); + auto sketch = compactThetaSketch::deserialize(is); + EXPECT_TRUE(sketch.isEmpty()); + EXPECT_FALSE(sketch.isEstimationMode()); + EXPECT_TRUE(sketch.getNumRetained() == 0); + EXPECT_TRUE(sketch.getTheta() == 1.0); + EXPECT_TRUE(sketch.getEstimate() == 0.0); + EXPECT_TRUE(sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(sketch.getUpperBound(1) == 0.0); +} + +TEST(ThetaSketch, DeserializeCompactV1EstimationFromJava) { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open( + inputPath + "theta_compact_estimation_from_java_v1.sk", std::ios::binary); + auto sketch = compactThetaSketch::deserialize(is); + EXPECT_FALSE(sketch.isEmpty()); + EXPECT_TRUE(sketch.isEstimationMode()); + EXPECT_TRUE(sketch.isOrdered()); + EXPECT_TRUE(sketch.getNumRetained() == 4342); + EXPECT_TRUE(sketch.getTheta() == Approx(0.531700444213199).margin(1e-10)); + EXPECT_TRUE(sketch.getEstimate() == Approx(8166.25234614053).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(2) == Approx(7996.956955317471).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(2) == Approx(8339.090301078124).margin(1e-10)); + + // the same construction process in Java must have produced exactly the same + // sketch + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + const int n = 8192; + for (int i = 0; i < n; i++) + update_sketch.update(i); + EXPECT_TRUE(sketch.getNumRetained() == update_sketch.getNumRetained()); + EXPECT_TRUE( + sketch.getTheta() == Approx(update_sketch.getTheta()).margin(1e-10)); + EXPECT_TRUE( + sketch.getEstimate() == + Approx(update_sketch.getEstimate()).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(1) == + Approx(update_sketch.getLowerBound(1)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(1) == + Approx(update_sketch.getUpperBound(1)).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(2) == + Approx(update_sketch.getLowerBound(2)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(2) == + Approx(update_sketch.getUpperBound(2)).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(3) == + Approx(update_sketch.getLowerBound(3)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(3) == + Approx(update_sketch.getUpperBound(3)).margin(1e-10)); + compactThetaSketch compact_sketch = update_sketch.compact(); + // the sketches are ordered, so the iteration sequence must match exactly + auto iter = sketch.begin(); + for (const auto& key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } +} + +TEST(ThetaSketch, DeserializeCompactV2EstimationFromJava) { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open( + inputPath + "theta_compact_estimation_from_java_v2.sk", std::ios::binary); + auto sketch = compactThetaSketch::deserialize(is); + EXPECT_FALSE(sketch.isEmpty()); + EXPECT_TRUE(sketch.isEstimationMode()); + EXPECT_TRUE(sketch.isOrdered()); + EXPECT_TRUE(sketch.getNumRetained() == 4342); + EXPECT_TRUE(sketch.getTheta() == Approx(0.531700444213199).margin(1e-10)); + EXPECT_TRUE(sketch.getEstimate() == Approx(8166.25234614053).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(2) == Approx(7996.956955317471).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(2) == Approx(8339.090301078124).margin(1e-10)); + + // the same construction process in Java must have produced exactly the same + // sketch + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + const int n = 8192; + for (int i = 0; i < n; i++) + update_sketch.update(i); + EXPECT_TRUE(sketch.getNumRetained() == update_sketch.getNumRetained()); + EXPECT_TRUE( + sketch.getTheta() == Approx(update_sketch.getTheta()).margin(1e-10)); + EXPECT_TRUE( + sketch.getEstimate() == + Approx(update_sketch.getEstimate()).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(1) == + Approx(update_sketch.getLowerBound(1)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(1) == + Approx(update_sketch.getUpperBound(1)).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(2) == + Approx(update_sketch.getLowerBound(2)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(2) == + Approx(update_sketch.getUpperBound(2)).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(3) == + Approx(update_sketch.getLowerBound(3)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(3) == + Approx(update_sketch.getUpperBound(3)).margin(1e-10)); + compactThetaSketch compact_sketch = update_sketch.compact(); + // the sketches are ordered, so the iteration sequence must match exactly + auto iter = sketch.begin(); + for (const auto& key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } +} + +TEST(ThetaSketch, DerializeDeserializeStreamAndBytesEquivalence) { + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + const int n = 8192; + for (int i = 0; i < n; i++) + update_sketch.update(i); + + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + auto compact_sketch = update_sketch.compact(); + compact_sketch.serialize(s); + auto bytes = compact_sketch.serialize(); + EXPECT_TRUE(bytes.size() == static_cast(s.tellp())); + EXPECT_TRUE(bytes.size() == compact_sketch.getSerializedSizeBytes()); + for (size_t i = 0; i < bytes.size(); ++i) { + EXPECT_TRUE(((char*)bytes.data())[i] == (char)s.get()); + } + + s.seekg(0); // rewind + compactThetaSketch deserialized_sketch1 = compactThetaSketch::deserialize(s); + compactThetaSketch deserialized_sketch2 = + compactThetaSketch::deserialize(bytes.data(), bytes.size()); + EXPECT_TRUE(bytes.size() == static_cast(s.tellg())); + EXPECT_TRUE(deserialized_sketch2.isEmpty() == deserialized_sketch1.isEmpty()); + EXPECT_TRUE( + deserialized_sketch2.isOrdered() == deserialized_sketch1.isOrdered()); + EXPECT_TRUE( + deserialized_sketch2.getNumRetained() == + deserialized_sketch1.getNumRetained()); + EXPECT_TRUE( + deserialized_sketch2.getTheta() == deserialized_sketch1.getTheta()); + EXPECT_TRUE( + deserialized_sketch2.getEstimate() == deserialized_sketch1.getEstimate()); + EXPECT_TRUE( + deserialized_sketch2.getLowerBound(1) == + deserialized_sketch1.getLowerBound(1)); + EXPECT_TRUE( + deserialized_sketch2.getUpperBound(1) == + deserialized_sketch1.getUpperBound(1)); + // the sketches are ordered, so the iteration sequence must match exactly + auto iter = deserialized_sketch1.begin(); + for (auto key : deserialized_sketch2) { + EXPECT_TRUE(*iter == key); + ++iter; + } +} + +TEST(ThetaSketch, DeserializeEmptyBufferOverrun) { + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + auto bytes = update_sketch.compact().serialize(); + EXPECT_TRUE(bytes.size() == 8); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), bytes.size() - 1), + VeloxUserError); +} + +TEST(ThetaSketch, DeserializeSingleItemBufferOverrun) { + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + update_sketch.update(1); + auto bytes = update_sketch.compact().serialize(); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), 7), VeloxUserError); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), bytes.size() - 1), + VeloxUserError); +} + +TEST(ThetaSketch, DeserializeExactModeBufferOverrun) { + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + for (int i = 0; i < 1000; ++i) + update_sketch.update(i); + auto bytes = update_sketch.compact().serialize(); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), 7), VeloxUserError); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), 8), VeloxUserError); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), 16), VeloxUserError); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), bytes.size() - 1), + VeloxUserError); +} + +TEST(ThetaSketch, DeserializeEstimationModeBufferOverrun) { + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + for (int i = 0; i < 10000; ++i) + update_sketch.update(i); + auto bytes = update_sketch.compact().serialize(); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), 7), VeloxUserError); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), 8), VeloxUserError); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), 16), VeloxUserError); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), 24), VeloxUserError); + EXPECT_THROW( + compactThetaSketch::deserialize(bytes.data(), bytes.size() - 1), + VeloxUserError); +} + +TEST(ThetaSketch, ConversionConstructorAndWrappedCompact) { + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + const int n = 8192; + for (int i = 0; i < n; i++) + update_sketch.update(i); + + // unordered + auto unordered_compact1 = update_sketch.compact(false); + compactThetaSketch unordered_compact2(update_sketch, false); + auto it = unordered_compact1.begin(); + for (auto entry : unordered_compact2) { + EXPECT_TRUE(*it == entry); + ++it; + } + + // ordered + auto ordered_compact1 = update_sketch.compact(); + compactThetaSketch ordered_compact2(update_sketch, true); + it = ordered_compact1.begin(); + for (auto entry : ordered_compact2) { + EXPECT_TRUE(*it == entry); + ++it; + } + + // wrapped compact + auto bytes = ordered_compact1.serialize(); + auto ordered_compact3 = + wrappedCompactThetaSketch::wrap(bytes.data(), bytes.size()); + it = ordered_compact1.begin(); + for (auto entry : ordered_compact3) { + EXPECT_TRUE(*it == entry); + ++it; + } + EXPECT_TRUE(ordered_compact3.getEstimate() == ordered_compact1.getEstimate()); + EXPECT_TRUE( + ordered_compact3.getLowerBound(1) == ordered_compact1.getLowerBound(1)); + EXPECT_TRUE( + ordered_compact3.getUpperBound(1) == ordered_compact1.getUpperBound(1)); + EXPECT_TRUE( + ordered_compact3.isEstimationMode() == + ordered_compact1.isEstimationMode()); + EXPECT_TRUE(ordered_compact3.getTheta() == ordered_compact1.getTheta()); + + // seed mismatch + EXPECT_THROW( + wrappedCompactThetaSketch::wrap(bytes.data(), bytes.size(), 0), + VeloxUserError); +} + +TEST(ThetaSketch, WrapCompactV1EmptyFromJava) { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open( + inputPath + "theta_compact_empty_from_java_v1.sk", + std::ios::binary | std::ios::ate); + + std::vector buf; + if (is) { + auto size = is.tellg(); + buf.reserve(size); + buf.assign(size, 0); + is.seekg(0, std::ios_base::beg); + is.read((char*)(buf.data()), buf.size()); + } + + auto sketch = wrappedCompactThetaSketch::wrap(buf.data(), buf.size()); + EXPECT_TRUE(sketch.isEmpty()); + EXPECT_FALSE(sketch.isEstimationMode()); + EXPECT_TRUE(sketch.getNumRetained() == 0); + EXPECT_TRUE(sketch.getTheta() == 1.0); + EXPECT_TRUE(sketch.getEstimate() == 0.0); + EXPECT_TRUE(sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(sketch.getUpperBound(1) == 0.0); +} + +TEST(ThetaSketch, WrapCompactV2EmptyFromJava) { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open( + inputPath + "theta_compact_empty_from_java_v2.sk", + std::ios::binary | std::ios::ate); + + std::vector buf; + if (is) { + auto size = is.tellg(); + buf.reserve(size); + buf.assign(size, 0); + is.seekg(0, std::ios_base::beg); + is.read((char*)(buf.data()), buf.size()); + } + + auto sketch = wrappedCompactThetaSketch::wrap(buf.data(), buf.size()); + EXPECT_TRUE(sketch.isEmpty()); + EXPECT_FALSE(sketch.isEstimationMode()); + EXPECT_TRUE(sketch.getNumRetained() == 0); + EXPECT_TRUE(sketch.getTheta() == 1.0); + EXPECT_TRUE(sketch.getEstimate() == 0.0); + EXPECT_TRUE(sketch.getLowerBound(1) == 0.0); + EXPECT_TRUE(sketch.getUpperBound(1) == 0.0); +} + +TEST(ThetaSketch, WrapCompactV1EstimationFromJava) { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open( + inputPath + "theta_compact_estimation_from_java_v1.sk", + std::ios::binary | std::ios::ate); + std::vector buf; + if (is) { + auto size = is.tellg(); + buf.reserve(size); + buf.assign(size, 0); + is.seekg(0, std::ios_base::beg); + is.read((char*)(buf.data()), buf.size()); + } + + auto sketch = wrappedCompactThetaSketch::wrap(buf.data(), buf.size()); + EXPECT_FALSE(sketch.isEmpty()); + EXPECT_TRUE(sketch.isEstimationMode()); + // EXPECT_TRUE(sketch.isOrdered()); // v1 may not be ordered + EXPECT_TRUE(sketch.getNumRetained() == 4342); + EXPECT_TRUE(sketch.getTheta() == Approx(0.531700444213199).margin(1e-10)); + EXPECT_TRUE(sketch.getEstimate() == Approx(8166.25234614053).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(2) == Approx(7996.956955317471).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(2) == Approx(8339.090301078124).margin(1e-10)); + + // the same construction process in Java must have produced exactly the same + // sketch + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + const int n = 8192; + for (int i = 0; i < n; i++) + update_sketch.update(i); + EXPECT_TRUE(sketch.getNumRetained() == update_sketch.getNumRetained()); + EXPECT_TRUE( + sketch.getTheta() == Approx(update_sketch.getTheta()).margin(1e-10)); + EXPECT_TRUE( + sketch.getEstimate() == + Approx(update_sketch.getEstimate()).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(1) == + Approx(update_sketch.getLowerBound(1)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(1) == + Approx(update_sketch.getUpperBound(1)).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(2) == + Approx(update_sketch.getLowerBound(2)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(2) == + Approx(update_sketch.getUpperBound(2)).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(3) == + Approx(update_sketch.getLowerBound(3)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(3) == + Approx(update_sketch.getUpperBound(3)).margin(1e-10)); + compactThetaSketch compact_sketch = update_sketch.compact(); + // the sketches are ordered, so the iteration sequence must match exactly + auto iter = sketch.begin(); + for (const auto key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } +} + +TEST(ThetaSketch, WrapCompactV2EstimationFromJava) { + std::ifstream is; + is.exceptions(std::ios::failbit | std::ios::badbit); + is.open( + inputPath + "theta_compact_estimation_from_java_v2.sk", + std::ios::binary | std::ios::ate); + std::vector buf; + if (is) { + auto size = is.tellg(); + buf.reserve(size); + buf.assign(size, 0); + is.seekg(0, std::ios_base::beg); + is.read((char*)(buf.data()), buf.size()); + } + + auto sketch = wrappedCompactThetaSketch::wrap(buf.data(), buf.size()); + EXPECT_FALSE(sketch.isEmpty()); + EXPECT_TRUE(sketch.isEstimationMode()); + // EXPECT_TRUE(sketch.isOrdered()); // v1 may not be ordered + EXPECT_TRUE(sketch.getNumRetained() == 4342); + EXPECT_TRUE(sketch.getTheta() == Approx(0.531700444213199).margin(1e-10)); + EXPECT_TRUE(sketch.getEstimate() == Approx(8166.25234614053).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(2) == Approx(7996.956955317471).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(2) == Approx(8339.090301078124).margin(1e-10)); + + // the same construction process in Java must have produced exactly the same + // sketch + updateThetaSketch update_sketch = updateThetaSketch::builder().build(); + const int n = 8192; + for (int i = 0; i < n; i++) + update_sketch.update(i); + EXPECT_TRUE(sketch.getNumRetained() == update_sketch.getNumRetained()); + EXPECT_TRUE( + sketch.getTheta() == Approx(update_sketch.getTheta()).margin(1e-10)); + EXPECT_TRUE( + sketch.getEstimate() == + Approx(update_sketch.getEstimate()).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(1) == + Approx(update_sketch.getLowerBound(1)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(1) == + Approx(update_sketch.getUpperBound(1)).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(2) == + Approx(update_sketch.getLowerBound(2)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(2) == + Approx(update_sketch.getUpperBound(2)).margin(1e-10)); + EXPECT_TRUE( + sketch.getLowerBound(3) == + Approx(update_sketch.getLowerBound(3)).margin(1e-10)); + EXPECT_TRUE( + sketch.getUpperBound(3) == + Approx(update_sketch.getUpperBound(3)).margin(1e-10)); + compactThetaSketch compact_sketch = update_sketch.compact(); + // the sketches are ordered, so the iteration sequence must match exactly + auto iter = sketch.begin(); + for (const auto key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } +} + +TEST(ThetaSketch, SerializeDeserializeSmallCompressed) { + auto update_sketch = updateThetaSketch::builder().build(); + for (int i = 0; i < 10; i++) + update_sketch.update(i); + auto compact_sketch = update_sketch.compact(); + + auto bytes = compact_sketch.serializeCompressed(); + EXPECT_TRUE(bytes.size() == compact_sketch.getSerializedSizeBytes(true)); + { // deserialize bytes + auto deserialized_sketch = + compactThetaSketch::deserialize(bytes.data(), bytes.size()); + EXPECT_TRUE( + deserialized_sketch.getNumRetained() == + compact_sketch.getNumRetained()); + EXPECT_TRUE(deserialized_sketch.getTheta() == compact_sketch.getTheta()); + auto iter = deserialized_sketch.begin(); + for (const auto key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } + } + { // wrap bytes + auto wrapped_sketch = + wrappedCompactThetaSketch::wrap(bytes.data(), bytes.size()); + EXPECT_TRUE( + wrapped_sketch.getNumRetained() == compact_sketch.getNumRetained()); + EXPECT_TRUE(wrapped_sketch.getTheta() == compact_sketch.getTheta()); + auto iter = wrapped_sketch.begin(); + for (const auto key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } + } + + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + compact_sketch.serializeCompressed(s); + EXPECT_TRUE( + static_cast(s.tellp()) == + compact_sketch.getSerializedSizeBytes(true)); + auto deserialized_sketch = compactThetaSketch::deserialize(s); + EXPECT_TRUE( + deserialized_sketch.getNumRetained() == compact_sketch.getNumRetained()); + EXPECT_TRUE(deserialized_sketch.getTheta() == compact_sketch.getTheta()); + auto iter = deserialized_sketch.begin(); + for (const auto key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } +} + +TEST(ThetaSketch, SerializeDeserializeCompressed) { + auto update_sketch = updateThetaSketch::builder().build(); + for (int i = 0; i < 10000; i++) + update_sketch.update(i); + auto compact_sketch = update_sketch.compact(); + + auto bytes = compact_sketch.serializeCompressed(); + EXPECT_TRUE(bytes.size() == compact_sketch.getSerializedSizeBytes(true)); + { // deserialize bytes + auto deserialized_sketch = + compactThetaSketch::deserialize(bytes.data(), bytes.size()); + EXPECT_TRUE( + deserialized_sketch.getNumRetained() == + compact_sketch.getNumRetained()); + EXPECT_TRUE(deserialized_sketch.getTheta() == compact_sketch.getTheta()); + auto iter = deserialized_sketch.begin(); + for (const auto key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } + } + { // wrap bytes + auto wrapped_sketch = + wrappedCompactThetaSketch::wrap(bytes.data(), bytes.size()); + EXPECT_TRUE( + wrapped_sketch.getNumRetained() == compact_sketch.getNumRetained()); + EXPECT_TRUE(wrapped_sketch.getTheta() == compact_sketch.getTheta()); + auto iter = wrapped_sketch.begin(); + for (const auto key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } + } + + std::stringstream s(std::ios::in | std::ios::out | std::ios::binary); + compact_sketch.serializeCompressed(s); + EXPECT_TRUE( + static_cast(s.tellp()) == + compact_sketch.getSerializedSizeBytes(true)); + auto deserialized_sketch = compactThetaSketch::deserialize(s); + EXPECT_TRUE( + deserialized_sketch.getNumRetained() == compact_sketch.getNumRetained()); + EXPECT_TRUE(deserialized_sketch.getTheta() == compact_sketch.getTheta()); + auto iter = deserialized_sketch.begin(); + for (const auto key : compact_sketch) { + EXPECT_TRUE(*iter == key); + ++iter; + } +} + +// The sketch reaches capacity for the first time at 2 * K * 15/16, +// but at that point it is still in exact mode, so the serialized size is not +// the maximum (theta in not serialized in the exact mode). So we need to catch +// the second time, but some updates will be ignored in the estimation mode, so +// we update more than enough times keeping track of the maximum. Potentially +// the exact number of updates to reach the peak can be figured out given this +// particular sequence, but not assuming that might be even better (say, in case +// we change the load factor or hash function or just out of principle not to +// rely on implementation details too much). +TEST(ThetaSketch, maxSerializedSize) { + const uint8_t lg_k = 10; + auto sketch = updateThetaSketch::builder().set_lg_k(lg_k).build(); + int value = 0; + + // this will go over the first peak, which is not the highest + for (int i = 0; i < (1 << lg_k) * 2; ++i) + sketch.update(value++); + + // this will to over the second peak keeping track of the max size + size_t max_size_bytes = 0; + for (int i = 0; i < (1 << lg_k) * 2; ++i) { + sketch.update(value++); + auto bytes = sketch.compact().serialize(); + max_size_bytes = std::max(max_size_bytes, bytes.size()); + } + EXPECT_TRUE( + max_size_bytes == compactThetaSketch::getMaxSerializedSizeBytes(lg_k)); +} + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/tests/ThetaUnionTest.cpp b/velox/external/theta/tests/ThetaUnionTest.cpp new file mode 100644 index 00000000000..5740f33abc4 --- /dev/null +++ b/velox/external/theta/tests/ThetaUnionTest.cpp @@ -0,0 +1,164 @@ +/* +* Copyright (c) Facebook, Inc. and its affiliates. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. + */ + +// Adapted from Apache DataSketches + +#include "velox/external/theta/ThetaUnion.h" +#include "TestUtils.h" + +#include +#include + +namespace facebook::velox::common::theta { + +TEST(ThetaUnion, empty) { + updateThetaSketch sketch1 = updateThetaSketch::builder().build(); + ThetaUnion u = ThetaUnion::builder().build(); + compactThetaSketch sketch2 = u.getResult(); + EXPECT_TRUE(sketch2.getNumRetained() == 0); + EXPECT_TRUE(sketch2.isEmpty()); + EXPECT_FALSE(sketch2.isEstimationMode()); + + u.update(sketch1); + sketch2 = u.getResult(); + EXPECT_TRUE(sketch2.getNumRetained() == 0); + EXPECT_TRUE(sketch2.isEmpty()); + EXPECT_FALSE(sketch2.isEstimationMode()); +} + +TEST(ThetaUnion, nonEmptyNoRetainedKeys) { + updateThetaSketch update_sketch = + updateThetaSketch::builder().setP(0.001f).build(); + update_sketch.update(1); + ThetaUnion u = ThetaUnion::builder().build(); + u.update(update_sketch); + compactThetaSketch sketch = u.getResult(); + EXPECT_TRUE(sketch.getNumRetained() == 0); + EXPECT_FALSE(sketch.isEmpty()); + EXPECT_TRUE(sketch.isEstimationMode()); + EXPECT_TRUE(sketch.getTheta() == Approx(0.001).margin(1e-10)); +} + +TEST(ThetaUnion, exactModeHalfOverlap) { + auto sketch1 = updateThetaSketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) + sketch1.update(value++); + + auto sketch2 = updateThetaSketch::builder().build(); + value = 500; + for (int i = 0; i < 1000; i++) + sketch2.update(value++); + + auto u = ThetaUnion::builder().build(); + u.update(sketch1); + u.update(sketch2); + auto sketch3 = u.getResult(); + EXPECT_FALSE(sketch3.isEmpty()); + EXPECT_FALSE(sketch3.isEstimationMode()); + EXPECT_TRUE(sketch3.getEstimate() == 1500.0); + + u.reset(); + sketch3 = u.getResult(); + EXPECT_TRUE(sketch3.getNumRetained() == 0); + EXPECT_TRUE(sketch3.isEmpty()); + EXPECT_FALSE(sketch3.isEstimationMode()); +} + +TEST(ThetaUnion, exactModeHalfOverlapWrappedCompact) { + auto sketch1 = updateThetaSketch::builder().build(); + int value = 0; + for (int i = 0; i < 1000; i++) + sketch1.update(value++); + auto bytes1 = sketch1.compact().serialize(); + + auto sketch2 = updateThetaSketch::builder().build(); + value = 500; + for (int i = 0; i < 1000; i++) + sketch2.update(value++); + auto bytes2 = sketch2.compact().serialize(); + + auto u = ThetaUnion::builder().build(); + u.update(wrappedCompactThetaSketch::wrap(bytes1.data(), bytes1.size())); + u.update(wrappedCompactThetaSketch::wrap(bytes2.data(), bytes2.size())); + compactThetaSketch sketch3 = u.getResult(); + EXPECT_FALSE(sketch3.isEmpty()); + EXPECT_FALSE(sketch3.isEstimationMode()); + EXPECT_TRUE(sketch3.getEstimate() == 1500.0); +} + +TEST(ThetaUnion, estimationModeHalfOverlap) { + auto sketch1 = updateThetaSketch::builder().build(); + int value = 0; + for (int i = 0; i < 10000; i++) + sketch1.update(value++); + + auto sketch2 = updateThetaSketch::builder().build(); + value = 5000; + for (int i = 0; i < 10000; i++) + sketch2.update(value++); + + auto u = ThetaUnion::builder().build(); + u.update(sketch1); + u.update(sketch2); + auto sketch3 = u.getResult(); + EXPECT_FALSE(sketch3.isEmpty()); + EXPECT_TRUE(sketch3.isEstimationMode()); + EXPECT_TRUE(sketch3.getEstimate() == Approx(15000).margin(15000 * 0.01)); + + u.reset(); + sketch3 = u.getResult(); + EXPECT_TRUE(sketch3.getNumRetained() == 0); + EXPECT_TRUE(sketch3.isEmpty()); + EXPECT_FALSE(sketch3.isEstimationMode()); +} + +TEST(ThetaUnion, seedMismatch) { + updateThetaSketch sketch = updateThetaSketch::builder().build(); + sketch.update(1); // non-empty should not be ignored + ThetaUnion u = ThetaUnion::builder().setSeed(123).build(); + EXPECT_THROW(u.update(sketch), VeloxRuntimeError); +} + +TEST(ThetaUnion, largerK) { + auto update_sketch1 = updateThetaSketch::builder().set_lg_k(14).build(); + for (int i = 0; i < 16384; ++i) + update_sketch1.update(i); + + auto update_sketch2 = updateThetaSketch::builder().set_lg_k(14).build(); + for (int i = 0; i < 26384; ++i) + update_sketch2.update(i); + + auto update_sketch3 = updateThetaSketch::builder().set_lg_k(14).build(); + for (int i = 0; i < 86384; ++i) + update_sketch3.update(i); + + auto union1 = ThetaUnion::builder().set_lg_k(16).build(); + union1.update(update_sketch2); + union1.update(update_sketch1); + union1.update(update_sketch3); + auto result1 = union1.getResult(); + EXPECT_TRUE(result1.getEstimate() == update_sketch3.getEstimate()); + + auto union2 = ThetaUnion::builder().set_lg_k(16).build(); + union2.update(update_sketch1); + union2.update(update_sketch3); + union2.update(update_sketch2); + auto result2 = union2.getResult(); + EXPECT_TRUE(result2.getEstimate() == update_sketch3.getEstimate()); +} + +} // namespace facebook::velox::common::theta diff --git a/velox/external/theta/tests/test_sketch_files/theta_compact_empty_from_java_v1.sk b/velox/external/theta/tests/test_sketch_files/theta_compact_empty_from_java_v1.sk new file mode 100644 index 0000000000000000000000000000000000000000..e8d50a7204e41a29183e7e0f27b7ade29905610b GIT binary patch literal 24 TcmZQ(WM*KHV}OJI5Ks>QCQk>R literal 0 HcmV?d00001 diff --git a/velox/external/theta/tests/test_sketch_files/theta_compact_empty_from_java_v2.sk b/velox/external/theta/tests/test_sketch_files/theta_compact_empty_from_java_v2.sk new file mode 100644 index 0000000000000000000000000000000000000000..bf630a7b12c273258f61decd9f20189f7c438226 GIT binary patch literal 8 PcmZQ%VrF2FJ2M#o0<{5z literal 0 HcmV?d00001 diff --git a/velox/external/theta/tests/test_sketch_files/theta_compact_estimation_from_java_v1.sk b/velox/external/theta/tests/test_sketch_files/theta_compact_estimation_from_java_v1.sk new file mode 100644 index 0000000000000000000000000000000000000000..eb6b271a155927f6d35445f38ebb83a3d0e9f6c5 GIT binary patch literal 34760 zcmWKPXFL@S14i$4w|lR<_mWa1BPwKMkCY^&j3P6ekcjL(BVU;u5%uJC{6FhJ9- z5A#cXJb>N9_dPyQ6hOq~fyLzlI$+B3Az<2p7oei5zaQhp3!o(|iYLV$2M9%#NBW%P z2RI)n@X1mI0J&zr2OpRT0wp!PIc z3GkLTLuF%68PGJdZ*!ER3W(t^nl$2+B61~j&IYZpc711JUXUl+A*0`8Rg z<0!i~0Uwghj02Iz0N<9HFNAaxz+}mM8P7vAz!SbKW!*S)z?2d`g3Pf1oRduE`f$<$ zFjZnVGc#rhxTs71F3E2LFtD21$yK%k{C!Y)gScQbGz|hgWf6h(8y^7%SLbRTe;op3Y+Q|f9`^;%dNUaeoI`e)o%@H7o*d6{IQI{M-a+HTkalD(nD0sfyZ7LVp9owpRp-xAp;^ zj8o?&7P)}nvcG`;?ty{dyA1Kq+o8bl2?4=rSr{-{uUP%rX*lrvRLS1C1}v~R>X(eR zEfJXFCy^PgO#>EL>Wh^Ka)7{s*=b#04)E8IP;uQgKH$U}cO0gn!N?w;|H z2Xei52*0zT2t=kGl|S;l3`{OL`<0=16j*58Q1A>X&QcIeaK)Z8ikyAakK#5c+=!&f)5dJ5>8JFh@470o``HA}h zP||qEFBSa&i01LRqf_qz)Tk`G7^Ch9wAU|eIT!_YUXEH? zcrpgOd3*IzV$XlTvS$cBCgrk88Mjb6m_mVht((CyF%tNRcP-b!2hkvq-@MCcq z(r+t&v*hLSx*j*?< z7MIu{fz|xw1=mN*-gu)AFS$A#CD*d1^SFerwpV>7zT$c=LHjZySVJ%!S&8s0lqC2?4FR6bY`rW}bCT(HR2GMytm(Hg^?YSDXl2?Kt*1EG9Syl{Wi>-hRh@S<0 zE17tJVTyw~-b3)S+7cjq9?mAwOA@sE)GPkF$wd%gQljHJS{)QcGaIVa)d2yreeaj# z>Va-7^1;GJZh`*s75Sva8iC+n1urPd8-pg|zHZ!kZ3^>dVm17c80KP zo*=l^Jlpf_L(r^Ekv|{o5vZQUN%(622$X?cG*2t`21#Cju9cSN0}>s2`X+`P0;=FY zerb3#6jXJ%kb>He0=52FqIGu0gHEP(LA-9Kf=+`>IxnZEfqE9#g39w=fW~k5i%fXG z0@c|@9J+@VgSgCIX^2c0gO0I++IOy(gD@tzxQVcGkhJ^{GGU++M7>Cge)X*$1nrc7 zoRVn->F6jz$}cy9o@Lvd@A}gSim(!w?b>SsT{gLp`07^+X#Ue18DM%lsFPKC+6&PG zIP#l=J%xx)k5)mwM|9NXjI8(@JI-v>%#BlI|V>F<$=t4A`0gy(;M=kS)3vdd1ry zBm|i%_GBOAW^Oa3Wp@D5G_Q{{e|rF0o7dxdyM6!~Um~tI;EzCmzP%RYQwD)o3JN8@ z@WR2jQs2wo8{`IWHYobWRq=qOj_GmD5-{MVZMWLTRxEIBQQnBkFfX_g_%sK6>Nt2x zaVY2D_X#j<{@w?@yp!O~zbf}yU!DS6Xo0F;H=hCRoGK4S8*75W zgPFhb$1a26^pmeloVCEGYA>&pm{@?fy9?@;Mr^<|F}7DF(FxpCnxpw;_yKr8*F8w& zwgjB1@etKdaI;^BlD=6#@C3Y6&}egn3;`={ zA4EWk!oY5=fc65vXz=NOheLJn81PL|N5QYFabStgp&i!QWbpT|@*OufGr$?j|7P|5 zv%$5eoK`N}%>nP-Aw zBNM*d3l@1-Y!T(g|v1ll-NB&Lkc@#R&;9|#QnNtOr#kZve4AioY}yDRDtIQhuWBs z6s>7_Azlt7Rg!mFrjZZA<30HUX>%NM5Cgnp9CsR$6Z`C)VV*GLA6v-n;Hwzqo?`B% zIJXp}$GH6_aatov6EN=E^5!#m*Q+k9on%)WdbSoI3z zSlQD#UBBy)m;xS+EUF=-88&2zjx&dFx4Un@-LZfK*zILMQ?`cauF^DF!Zwg_E?KQ> zS8O3&L0%Cvl=~3D1XxD%EhmWBO~b5~Z!VCfwfK#N^RAG)=q=1Z$0LaUeXX@Y_G3r| zrMJ~hJ{WSYX*4FcDGqXDIxg_FRVoBitYLf`m<=KAJ^s)b@&Xc2_%i?Wy*x<2=S7uQ z$b1OjltCav{1t=*rAqEe6hc0H`IrWlE`pqK)e8wdTMW6ov}3m~^crHQkr=CPS`OiV z=4BruUjdQxD7s~ErxK!3eDlm(#Tv-H3+62_$!5szfY(rd+T{LEYv*P+QeiR3w6;6>9eWAK}Ghn+ZX$Z&?w=_=cn~4P(}=>5@g=*xENMbx8H(EP`I@iM(a(D0A#hsy54(7lek{TW|`q0|1;^8Nv5 zq2sPE5<`ATK$S8dUZ;*rL9HX-@>Q8BLECU=hDA5kp#i~N?#qq3Q0#**pLCwyfZj0a zQyPdchW5TXg}IBo15LZNZZ`Sl4s_XtEA|TB9NMVNyl|t-4vO1u{Ujp``h~(3R#;sQqOA%MPz(s6MFV zn%8g|^zzT`n>)IN(4PbJm1Orf(4oJFHw<{Hp`+y5?G8s6Trsqk2S`W+^Uz^*#&6FnLZ{g18MnAuksWuBfz$@Pe%x5(wOBn zr!NlE7*+nAx^xjXRru_~)dw0dGC^~OkE8{A{! zB)kBteLb*Ud1Vc@67l?UQ`0&uCMgcij}8iotg} z_HHDOU4S>7^oJyh%fcnJdp6_8W#I}_A~QYXa`1?SAItlF^6-S*2Iqt1OYrpYgU$_C z1-Sd{cgZrgBAh&zUD;==1pgM|jku$(3K!W(PnH}|hv#_xL_~bkgi|}R5GQVE!AtzS zUwf2XfxpqazM0{r3%~T`yTOk1O}K4jZTvf%oA8NCS$e3{+we2jE~;3~8N-+T^k1i3 zybC9kAU^0d-i1Spt;W3YW^e)Wp#O^qGdOr^HM}6{9^5W*=U>Fzd+;Z6uiR}XZQxV; zABRH-PH?{%nr*0;6I}gb^sE`m1%5VGLkc(j0A7fBh|IJa8Jb7yKBoVhdH;R0=kZv}s%SpM#SH{`q*x8Ul8OV`@(rY!ftb0kliDJ6b_ z_na#f{j)p-_fQ^hsA&HTSA(qZ-{%^IM^36f4Obe4J2X7|Fl9Ur$DeYWQx2Sk3p1Dr zQBL#l!1St{=FBDd%%H&Abk1-1HHPW2j^00TS9SFu*5n_!|85{U;s1V}a1xJ%qov5c9iGW8We94|bBW~0s^M7ks8 zmrc(g{#@{G(QOq&xSA+R-tm=0DBEu?5vL^);^)ume3sBeeAtp9pOV!=Jc#$^k2BRq z%$wcoNv+UEWGpWSX=PqRD145;A9d?G0+IRaMAq^x1U2vEhx!f^1i;J<$v$O;cxG|a zcb9I5c<%W%I+x!Wap|S}E!PESgv8zLAL7?t5Vhe|hD#UR5CsoDj+M51BAow0jy{_N zBPNfl<3+545fMJWJjP0c5wBXl3E%k}il8k-ul!jLL-@IFgpPcOLd-!nBDT2_5NRjn zd;5%%5pV0hr%D{lK!9$nZA}MfApE|%ym@MxiTKY9JkX_|jaWAo*ZyxT8?o&h1ne5j zLHM<;ymh+%5)n{8+UQqSfT%q7PTLUi8sQXMXa-@wM&ueLQu?(^5q}>|ziFaXAbR`l z-VArWMWFtETOPbzi|{;1|L{$-4)I}DU6p>j84={_BFvWPK-la%Ua_w0L5QUHjz*dH zBlg}`o-a)uL=3Muo;Viv5%DTgLhsGQC_=JY>c!=bZ-^IlDla08XAv)oE%BF+ts$Py zJSeip{X$&z)>+U5ZX>EJpL21mZzH5rHh(7b93fhZ-=EQU{f8L%op)*SH-Ou;Z>G3O z0?7T(QZk_MJCgf$15U52k%zkj+wHU|Mc_tj`#*cEOX7B+k6*6*MBzs2<$d1Tq;fw8 zfRg3$G;T1By8=2yyPlxsRB z&E1*1^d=%ynH!xJc;%9>3U{aCEpLZH6>b6EApR#l*STA#&lTiJ>T>@`F%!7pa)X=d zJNb=fVaOc_0K7`*y2VZR7s0<8c*qTI7jZvV@PxZ!GbwQr8^CQnJhf@z8N_V|LjP0y z5yd^W*Y5{2isc^SHuO5X|BO3Mcw$FQE`{4dDPhs>@FjQeScf?cP{-X8Xjo*>(7>(4 zQRb^3XyJAqiyiV@Y2k)|n)$I_t=vgR{6l?&Zth5P7fHv#e(q>os3Ed)otsT7Zg;=8 z!`)aJaI&Z45BJCK(mpXkTWCc5tsmiZ`{8eyA^mVuf5>oV0lW1ym7pA zxzx}V>GGFTs@wknX)xdiIy>Tve8rWq@;y5c`DtFa!+j+f*|Y54*nTS<3FG4%6?BS1 z0=Vn(DhY|mn?v?*lhzWEeoU%bglG!#NNWhn9xUAAf;NJ-5vj}lqp8yPpymzd6pyn z`_dHSwyKcLQ?DaeztC~-vH9%S^^s4GKLHa(tXg=wSv5gyBH-j_zUR-@U$^n-$I(-bBKH= zw2NfjLO`K42gnSGTaBXeAe607RNdq~4qeilV zCaM-lsJygUN14lPlf>O?g44jl5Ml6ST;=f{Ll zn<&;uc#IgTN>bkOVaa(^g_G)C3ibl(zLW9TVVFGXb?DMccDWA9+y;$1dUgYakXg|F zl3{?lEm)rUZ^01NBixa<79kqXG=ruvLISb652brRZ&O16Th?$}^ z4u^;M6wOhn)ZKGDLKdj+dZJooZ|qP>9`RcqjdrLii?AHNc01H;o_d&HxHIaCoogv> zzynorc<0Q!*2k!5oK>f=g&(Sa@@y3?;t8q^<{_p+3qu(f_6W!5hM@qR_g~A%g`<|Z zE4=@hM50_Nk#h5nktn~Ir`~E?aj0;(zAN@m9E#hsPAz@vrd!|)KXi9?Jvg>6rY0W^})Uo z6xg@2G!*t9>V%9|)rj#7O3=;yosRnq>fD!u?=8LaC^Tgb_%Le$Wq#+QUFGpb)O~{Y zOV#VYQ2z`q@!$LRQK8|cQX3kFD4zbIds*c`^yAG;UX7!?sYKGvr37Xkh@Iu{xRf3wS6|aRIW(!z9=u6=gU>E z-~nE=dAi!hCi^6MGAXO{c*j}vojAhm-JpoL%yL1P0hOKrPD{Qa^#!nKdCwU4vNNTsN62Y?>}R-`0fuN zGtv~@u9Y(z%rr+o;XH_?X4#^frm-@r=}zcxpSeP0OCF*FYsLHajQr5`37@|yGsDsP z*D`dxzeb=BEVX0$m7~%3^mUp~euzf1VG*af&&8vGAC4RiQHkhxSv)x%9jWMX$q}B4 zQVWxPdac)TmTe7OS z@@Ph1Ira-(y3&IFATyq(Q`m-fLMjKZee6PS=@5BH2i<6uSLssi3m?#t0dn#XhXHiG zQVCq7brd~xueZ+DH2YXauzqKs?GBOxmSD5YGkGnJ>!jNS+6du2$ z)YV)g8joX>i|L&!44$&58fOH$nLO_`rlL=_^7H7oW_T=JI?a>eRPHKy|1?kDPPnji ztr*X9-13P-pd^ofKq)UZR+8s%;OO+k4H+J8O(Q|f9~mC>-w=T%Gi4rlYfI~kcy%7p zRsHuGTiQJL3jW%7_ut@Aoub5de!I<6s}n!rUu?>gMP%t4#NOd~ckH{)x}6n|7k7L4 zT_qcyHwnOW?lEVcKlCy6?>?S9tw~qH2wxuYEE&{=Ua<}0k?wba=9~-RxhJ_2^?W*% z$K74P?QVPokHTDI3(w;m9ys`;<@WFt92dP%#)S%@|m1?HxDvmaI0KwfCm`py6^w= zBhNpGhTh@jPdpXo$eeBPFi+gYMJD^iD9@G8q4zUO$9c|gXD_WR&GGDK<{Kt@{N?HD zT5^@G0b^z|zs-*>ATW;Vp<)S}7>sqZXi@4M4nuL-t+gf+FxC}qpsb5zj76eBe=_Q~O3c&gIqX2N_J&A5Kjhrr{QuvU!+Vlrni7J9_aqqZ}c`|*)TBYXJ?5qR`XB}jJLsApb{}zgQ zHGVbdLT4BTdl#Mb3mb!Z*jI66>lB0O(zUf$u1Uf)d~(%WZAir^8|hzI-%P`}zR6Nm z=}O0(_?we0p`L^BIU#v1>#z_rs~Nd!4=ll;l^&K3$dzM)#r3Lu$0{)4=RBX=Xx3ve zyLWv*$+cjVU#{2tAiFWD=Jbb+UwSbymSs&(xcf0G+~qA_G>0)S-p+jr08e0?jmKJB zAd?um(Xj=yiYZLt{m%onjnkOd=_xsiH)b#vxn=6<1xpxgd)c$g&C8glbLA;!@*9}w zoCfOU&<%_UW^+B_%`Z&ehpB~y;cZMwwF=>;{2xrnb?&OM|MoEGpKGvyi31GZVY1DP z?-54!uEQCnfFsPm2P%}nJO40(ccgW9^nlp$72(L`7$DaEJa78r1SpoVe&BpJ0EXT3 z>_Qi)xzcY7=v9ijBtIcjmH|`2l7jTN!WIR&dCW9 z1sfM$EbY=v#R9bk7yS?ESea9HbHfM1*yH2%T#g6lus_Ye`fE*Jz?N^tBd;w;W6d&Q z7%mZctXt^)=dCX;VRNr+-zDKxunWAmul#pP1$&~Y)Uq({GIsd(AwP9N8*7C;|KgvN zA@*#UMPJwrJM46ej?%GQd+gn231QyX4p`ri@cTcepI~z;bZd?#0**>K&ADI!#e{w%y6FmA~2OsOl_V2pdHWUtG zzaE=`IMPP2%i__-Pi4liv-$tblFm+H^(ptaI<8G)pWoJ3XRLq69(-0f`O$D5>zja7 zh^Sb^-c*GiRa-7&Cp7%hR!;uH-v0}&b$h;xZFp2i-jmtGu1EZS{@WRV8%sjsJuiW9 z|4CIVZ(2ZbuRkz9x+bA;2tg_A?2GGXWp!`zH`^%V>vt+RWp)Voy^A zx02&H%-jnTsm=d)R?C`Ok07qZAhf0@T?hx#^9<*;xP&thwCasmQO2$QZLxKIsEso` zw=@`7p@TEH%<85K>)|XebyBot3~{$*H69FD-NIdXj;-58nBbg*8m9N`OmH%M=dDof z_Bi)T+16tBU2qd?_HUW}fjIXFkPyTx9Oo7CUics-9=B9>a5(CegnNUFtG)Z{84mDJ z{lYXS3umeVqueLt;I0&D2M*N~;I0jd+9$Uc;R013=cF!|;wp3Nwsx9paPb*C#=o8F zaQ3S7F$0BWTz|FFPVbW*T;e{-r8@8nu269;dFIkEj@yBVAX^RNe1f)%8goW*Q}?|4 za?`)zk_gW`PV@YSOUljb>h$@EGj=nlFGBZlz{*>tmneU6jjx;yGgu(JM`s+PJ`RKj zTXgcjJCDG38u7iGjm6;YItL_K0XV#Cihj~35fc8wJ&(h*O(x#HqU)>h`(yZj0+W^2 zR|N45H{*P+k%jPc-kqmU07URI`edirLs7gyxQR0<^Bg|=$eI5OUJ`#|eCVjN?INB% z?0rwEL=NwydvhPqe+iF!{>Xl4K@m?^Me5y~R>tp!LIlV^H1Uo?%R1*1wD46IUK3{C zY2&lvwyUI0=;K+TybZD2xA0;yt13^<+{5=>M0%?A-@{L*wMlm@JK*2Q_46GBIN}9@ zSm(9RxZ&%hK4!T@df;`geHu1Y^1}BgOlOF8KfzxtJIAl!{>pU8QKE=~4BBM@?CgLM| z&>5A1X?WDb%Giav9Q>u7{8Xi!eEdlF$oEh4`S|ol)uqfE1$cQI@dju_2|n=I-{UeF z|M%jX7CG%Qd^8=opR7}cSAAo+HWOWj_d1U6%zXU@@96JH^vtTnD{)&?IA5y9_v<23 zT|%1h$7NF&Z_G8}2floIame3>kIDBXcqFyq>lb#P8ozJDpEN4*5!GtPf2|w?nP;}+ z3-?_FXJR|>Zl80;A#2@uAoz}=OUDQN$Wf-lp#LYl)(t_2U&G(j^)!IcxZq8?LJ^N^AI~ z2Sc$d4(s^3DCmuAZR>dLD8)IIPaF7<8lmX5OPhGBsC)42``h>{Gkz20p1b&eV;5wo zR}b)WQhw1E2S@mw;Hlb;FF*pYkupE1h9catW{hg9U7^U}z7{Z4Y{#zCZJi+h8 zX_!YZgYa8{x3TObhp;8!X&NQPAt)LND+xa55HfZq!iB_75%}V~lPqLU6T;!5?e@Qg z2|7iZ8%4^JgtKMVH*8yE2wp#r@2lUGC1BsC*x4D&6ZWzeY^BdD5Nh?4>R%Tt5sn{7 zKTw?3A@tPh^zUZhBJ@b66v8jvCDAQ)B)_$CxRB-Dtqu(#yB2!E#ojUXjngu|_9Cmj0`!2>NI zkCb>sP^c078N206NOhg+OcV7Z2z@>3$1nL2ax*wjZ(yd>8vD zC7Uk@7jEuHsa56@_N59_!pmL~u63!U@B|bPX37b;#Pk9}{tXAyCsu`oS?#^Et3L_} z8iPoA)rBI$-t$Zg;IB6X!}OTr3dt1&KK|1u?>%fKTrxfv5t`FQ80OYpVhi*VvVOt4 zWj*@|Tf>7z{x1dzwH1Q}`C6X|RGr@9am!(X^%|tnCU%NIS`)VYnmR?8+<&%eBK99a zH8n!5?85@#u2;Fkw&fy09G{(J@pp~THrw|oiMT=7xcE83sbq_ARkct~N#r--hh=H` z@UdOOMwICF3L21DS7)lERSh8yCZ%pKZo`PDuGvU-LlDFdXOW?nP;TPY<{QuAH8Di~ ze_3K}V;Eu?>6ndB8J2kOPJJEsDv>CP{m-1fPb4lnpZK0XP9fI(D_mew7{sV!w?x~A z7(_+xI}hBhA0t+CA3vs@B}j}*+jTLv5GH083yHm*mL&$iG}gYYsX%nDY0X!sDH6}y zT>IW9p-fB^fT3)jXc8}u#%kJHUnY9fT^&;~*NOES)n7je>Jq=UFIjZU=@F3+x`ZEV z=o6WxvPwDI_lRv-E4!KswnUGOL^Ddc1F_x0GwLhYi3ocAbkpR83sGdg=NugDO7y?V z5}J*0BZfJmPkY>RCkky=*D7IRh}>%9e7`7(M0-buH`*qHIKcW8e>NzexcIVn-|TGx z5wu@GyxCbo%$ci7wwJCW`jT2c+bFyz1{^0EsqPRUW5zM*Rj6g}cO7;Vt_2j$Pss*I$XZ z0)L1J$pIJUtNs#Y4~?f{NB3A+;hm< z6iG4ad*@l%B+`FbVS<_4sicq86V2C;rIY@sFOF`cWRQA6W<`q&$;uW$iGK!p1<;gL+*9voc=1#+VAv|Wk26uI~AB<$U1DKaeZ*6D^UY4VS(c2UB#JlWn3 zn-dzKL|&v?F8~@<$#Vw*PX&54$TC`^xMd$La-}$7#XL-lyjAl1ibl%~vVg)kpF*|) z*~P~$b9Bs_4DTr$Gojm)zp~9KesEWE9PzY62GoPB@GD$TS;>o>EP{OA>Gz0CJ|h(7 zIP;hs-Pe2`r4U4hy84Q?)di6~Rz#gH_=S@BE)>jy`@+bNq6Aizlak18=c=QHo+gtI zcTy&o|D=&!9`X4A^U}$&gULU8EwjjqtJL7jBH3hTO$S42U^aQ?-D0<-LN3|bZ2Ukb z>Lqz48{;fnR4>8 z90`cOcncXK-n?_@*+MS;UFaq(*G&dh&e}fw*iYV?m6gD643RTFn|fK@93_X&@`n3m zjgtSW&0i-^jFZ48&c|%CJ+uUj45aBrUx=aOep&Qw#k5f zJBn#Ir$sfFeQg{v&<8dLJ1((KoaAl%K|qN)G4SD5{UBE%vAmN^ppW zsG4jyMbTaTkMmwXMJLEm`_Y+?l=FAu;;XqoQC@@FgDYGBu|rC1EJlka?6pafoZC{fK^q`0SEsJW&psc3`ep%Zm{RLd29V;5HeD&I6|vXM)I zdabbWkNSUdRE-J#$)fuTRNd(p=}x}N)YE*XB5uOFU`}c#XfiKqR`%+&gj*g?24r+LK1}9LPR&B?$>z`4hfiB|rx-zK7*UY~3 z-F`u(NAYaCA#$k_{*4d^vjVD)oKo(|>SF5ah|On-1trvvnSuXFzk5w}AI9}u@-CxZ zw~K=@dfrfNq4B4ma>}X2q(j-_z6$E}Wtso9tgER70t;I{d^J>cjibiY$QmkUISxN@ zwwCIpb@G^WR4tWQ{BQGiVjcB_X389CvV~fYE2&JXZ>64dL%l)KK2TSkE!xUmK2wME zJZZF%FI4ONdeLW{#?tB^VH#qytPCF*Oj+Bu=C%hbuA7{YS&3Kg%u^*%w z^1M27k1DzP#$SJtodK$;r2t`vVPh$ha%+kldW z(q#M|Y0tkx(C+(rmxwGNXrUH@=cAtDX!JmJ`CTUx%^--f8fH$Vb@Sgp41PeRd0iQK zY_K6pdnCFPV3%^1Mj!u=xOeP4jrchApbmMFHXgU}(EU)B<~5$1ZR322hDgDEtg=+1 zy<6%I6DwAxg;7;33*uC17PSvWciZ)7LqoEC>zjJCPD#HvQSdd)_MpZOR}H%_E!o1G+_2PV;K_xEBqg41a9_p<*)8q;X$Mh3qd#j=z6j4TVWmlqoGTzW8Lv)sp zWmM2+4L_@$imj%JmHD7MUGpzG?rsk)_f&ZzH0lElkq+a-kAM3A zUgO6}79+IxmR%3*Nu#u`9<_AVwNV=Q*pk!G(gdyV#4+wd<4M{aAT={`{y*Bt-YpPjOfOgPuQlN88B8nwYEz;J^soTi7UO-nv0I8E<4rb~Da zm!=zL`)=~)Dbty)-p<-zRp~0R0S~GxwCHww+U{4^_2@sDH+kd79q0#$I}__Coap`N zNoAU_E8RS8MlSiLE4{KsvE1Y8Bf61PU)%bS4_(RY_Lm&q$MiSeDb|EwKYB-01G9eE zj~=3kdzAC&3H?}5%Zs$lAo@3nsZLN~Fg+`5-pbJ>j2`@U`@yxxadbUzX2xxgr*xso zS=wxO3Z367W?1J^CcU(_ZTsuRm-OWT7?)+yE4t&p10_kSgl?P5Fv*cAr+-_VxA~e95aWtaT)o5um;qk#k-F}WWMHeDGC-L~ z2FSY5CK;psM}&oxHRvHuFw+w>TXHNwn5Pkn|u{+Uc^ zhascGI(GZ4)@?>`sqOvBdLu?LGoI4OeTQLn;)vPz`wm0Z{dO1I&zvz-!lf)*V#~P3 ze~3PgiiIB;mrT=6Smci|l-dOh!>4B$#?6SWR+sOLnEcPt*U!x} zVmwUpYtj}N;MN@W*0m)@$d!MJE5Ftlx@wc$zH&bqdtQaI@!`K2Uo|rcy&Ag=-Sg=( zE@uuHi_duz1$_?~6SsYLmU#foQxyW&aTmFm??mbjKk_4)KF1zzeLv)3-hcd#iN8%` zhU^AJRXP!wMn%j^mA6UE-$;_Y-4u;kpea+k1*bEolOR)dLu@8YS3IcQmye0KA%1ff zd5RezzL9IyCd8ccxnKY2hd6V~7a=MzFTwm2m?V1i_8ha~#DhZg=mn-xvWTO?nly8E zOH9wMR)zUlK&Rr}tv#YW=>C{lCpBKbr4z`!`jkvEQ3_&y`!(aatrf~NOJ0{mp(2@QUs}3! zJd0%too>xJ*B8(9xa4o}X#Y9WR`K6et*dFw)Hh|RMH(5*Yrn2eH=s(H`peTik~MWq zlbUlgPI4{G66+sX+pV2UQ4jcvk>dxZzOqU12i!;IDZ$2V4W419p5%wla=9_467nPb zl*)Ie*|P7?AlZQ5mWt9}k4$NXi&r>k;()PO95 zhj|rwt{~P~TdkS;1PBYled@h?0F-5T`(fzbQz&cy$48cW~E5o2PL*q;U%P)KXwO>^LOX2ft z6@C67*5C3k5TV=S3RzO9w1R16qv5a^YuldYN>Ww6p z8l$t(4wJ%qW_oafmh_zEZ17dbT_KBglrJ*E{*}!_a5XBvF?`AT78PN_^e$o@FWa&G z9$wADhJa7ZuQjvAr!&q3{BMvIA5v#2`DU1Pc2?`suODNq zva`L%P~}T3sbX!30lih$Zp#U56lskmp+ef+EZJbyelxJFd-RLdN!;E_|NW1Jr_2d7 z#dEWlKjz;l4k56`=T~^@pAy&ruYn$}AR^oPWqPfV0hPVGCghdyg~jG^bBbf#VY8j~ zPuRP0@v`?t9Zjs#PO=AVijAY~#n?7f$EN4fXW0pJ5Thjnadt+AmfBwS1-77(=WSJK z8MZ8wr=@gDhF!J#&w4IGp1pERGUV8pJX<^x+f7?mX4~CWo9IBQvek7ijCG$jVB5fU z8l_!su^kG@4VSIV*g%pDuXL;#yGWL+z;oY%t$y>K;u6)0?UiN^oq2ShZ9SIkSANrp zU8L7~@u})V_Gd3*h-%J5c1K*PmBEfT`>OWT%_pB8vyp~R39KJ}?3#IGTT-AuJEu~z z(`w0|?Q@p`{Vp5G&dyGov2qAyTZR6HgZ+lrjf>Dy0at0kE^Rmr8W z!&Xbbn5?C++ma9P{>~Zf3t~xvO@0~diaXKUf@8Vt6h%EeGCrRTEV9Da`MzRTyGRkx zSBu#0U0ORPPDO0njdLj;)MB=WE+0n0>n-~w3kC~(Xjg9?W{m{RA zj;-}l@uPhC3Y-7^$nn?WtL!7=|9Z&k8|*%Xrf-#Bf3ZU~iZ!Afw%M?rNV-qnU-pGX zH_28w7sn7HbbUw@!l?-oy4(I2#)&8nd%Md*aIRN0c;q{wIhIn-r?U0MI80o-fWU7*6SPLb)R@FLvDI%mY2lER%0>E?%CqNf|QYOkPHizuK8|K3^2`yH{k+ zS$bP$kyL$;^NrWF%z4d<^UnN{sYA0hC);uRV1{DLIZcfmP?T}uDBUTVZm)FZ9M3b7 zq8~ostOQ;{$^3rE3GDTNImY;Lzyq>}hm0qj!2*Y_-rXRM(pO%QKh7Z>D8VGlRxq3c zy*jRJa5q zPYI`Fv$lTt?`zI{#LJ*BlcgMc3FG{)%yQ1vI$!_l;0n%*p!MHB8frOPA!nrd>smNL zjmpX`pIbP=BfRyi|Fv-7iIw%?((N3#7c6)Cm`+Xs+o5vKrHjL5`l7ogw1=ZS<^0UbH*%ar9im(p%o8~}X@x)8+&vMMdm%QUha~$zQ zOuNXz5@&+*P4HaGGRHJwKD^m`jbl_I&&OH+$pNJ>mO8%w;esd@6 zapV>JBpUL7yrhj@fS4ng7e$wGd>IPk#rWGz82TW1!{c)PRw9wS4_u8b1Cn`oFIF2V znta6YF8uOW)SkrfZtop#l=R?uA)S#PZ(t-|gL&1;C&F}Id42W&F?61RSTIl&E=4F* z5{kTgd-q;ewvatDLWpEV7hU8zTO!Qh=0Z=@O(r99Ns;XNce#Suws9cI&^V>WwHYe^_&2RB9#8i zaVG*Yc9HiToaq3NrsTf{e>%X2Z!0SQCKHf3$H(fVu>ojm$CD#4Q+&kpB4GBq%}QtJMZn=H?SV&Ks(@!dUME+0sR2?Zo~K@d>H|h`J5Yb5 zAwaY2@yqJ_#sKk#3U`4ECIGYIH|TFKOaRTjC&xfXO#!JluYHVpVhNCh$|vXxSp($u zH}UDI_JG4Tka*5hXMi|?=tiTu0#rX~9*5kz0q|#67gefy0<6074;}P-0aV_zJ^f6h z0fGRYPd1|QfW+jkR+d&gpcuJRv{V-lxZ4YGdOMr|_{K=d4d%%OkD_7C#`hVDY31o5YUig>VWsB1AGhMlR~*--+>s;#uLft3O5 zGrip+bgBTAHzYjV)LsH^&65;cZodR@UytXzxzzyRe%;TvR^9;2l>f~HoooiM5|snj z(QSZzyA5+wl@5TLE%s?|P&dHNysxgI`#s?1Xj9z$haLdUCp0*}{tIABdg85tz!V_7 z99hHV&H$D~&WO)C`~fUE#SUOvHUN#1ERV7idw_rpQh-?SK49cFz^VBB5nzmKGT|8R z2(Yi0-(=?rFHm_gdoD0p5U7?YI z1FD`Z@Pc3bC(8;fa9fmkhOdSQRD+-uv;j0=i(L%9X^I8Rl#xm@+hPMDg4}-|?ef5k zH23}Qatgo~cimG{#FT(ZDZ#F`w(39|kmVRR^d-FjJ&Ke>jLy2vC{Z$a1&Vb=RvfNl?U+W z?(1>RD{mm%Z5Yz?-3QnK89c4L9|HV(xb>pNi%_5n#l}C$~FL(%Z zx*2%xW_8sA-VUHbjK1$^MGw%8|EHg4Wgk%b&1A`V+7NJEx2wpkdjxpaNLR6ZdJJel z750ieHvyd6x~v11ngqTdW+b7{e*W^fOO}fCkg~)K|XXyUznvV=v+VkczP@j6xh_z&c{Om z_4U{t(w3)z2uil$mUeO=()E)?cAD~_icS7kJXXq}d!Oz0SQIUg*z(_T^9wp4=un*h z4AU6YlvkR>FKhgyQ-=pA^p(t4Hy>Y+q#ncgM#;U4Py{^TzdRiNG}Y$IXPp^ z`xg%0+z^rU6hVMHI<+S6a?s$7XerLQU^I9Qs_%aO5e59}Q2ccRI|Xn83vW@Xpa}kc zx_C1Er5cz{!umCYYJnXHSBe8K>42B#w)OQEE`xz2YW$Fj0hsD4)D$3O0j7w?pT4GQ z39bt{o9j_$4PMZ^mtfOt4fcqfbk@CV0~R|Xq5Ue)1}rEnbrQC23myygxwsf&4@SFw zBYLP^1M}tmy;$ty2-e+c5;cc9fqxxMeK*{C9cdQC2Ft#6K%9N`2R#2liNkSU126bmc<2ut zg2d{S5?U_sLts%xU#nLm=gGH9-?6A*IaU2B4}_5L$G zCPJ9_^y+QMDn{~Zaz z1Fb$2q@y65<7gRVZ4|`ySk|sqb2LPuIX|P%FahFYu&py2{t)6qT9(N>NP+yIH~T3` zq(jav>l;_~WJ3(=UnlcB=0Ny2e9b?M7eV$)$p;%orI7j}l=~2(4)U=vs)e#v2T5Cx zNp%!#gOFO}X{mi35ZG_ISKD#jkd1i~I<}`DV(`)~-nM)Iau~)t^ml6*;`Xv*GpThR zl4|O2Bc!qj>3r+xFr&N-(R6803^@D;l3&#cerxatqC8%+OHyBl9GQOjoX0=}YEfI| zcf4N&TI)rXP6!Z%HrYC%GaSXBI5W7GpM?apLAcI(Z) zBa7{+Q_|4>-uS?D6Bx8wk`TJ?0f#Q_o3CIxkkHxQmF_}o#|q9J>WPLTS|7#YyY4|J=(?0cr&6IgwT4cXu6ag3J0tB)~ z9u|908ueJ_0t_}Fyo0&}fNcyvV4i&kgo%9v4Smpu!^Y3~+vSZTV3(XW2EW9hV7H1_ znOTp~Fiz^|gZC&|SYOf%aNATCW~y@INTn_r1`yR5%n+l%JX2g8T_mWm&gExxjJmv(#Zd^ zx+cs<^WVBvngJ}@$}ACb&l09+c6xJ5)fP6lR(G`OiYtt9y{{|z@J-nL)~WE?`~X?4?_;h&T5 z7M{c2``Y$N)mOk`Vz-KVOX+dSHt(7G1z;v@*_h}Ctz28&2km!-(Y`SpM61?&A@(2yjNh9e_=`ozWwoATQGY5q8L|g57xC8jJ-3y4|C7E4Eif2 z41dFL7tbbnq%|Go^rv-Q$C@0KZCEP64JW@QG~?^@QNA2o+7-Fm>u6}5pU4Z}>zyzJqZ z23~C3*L8$X_E&}9#a)MM9~+QRyX6KqYp9u5|LYEy^gen{;Hwv0Miad*dfywqr+w*M zqH+*?)`8IdVJZxMW5!F>O)?yQ9i37qsTU5{I<=Ji<$44>vEqjFKJ5vHk%TCtPI3Q-Ag@%z!P zapQ?_JGs{rFLYDk@UMqut|w%`F(s<%y&rSoSK6jMG)Hsc-t8jes}p(ffj@+Ewyj0* z=T?|@t%zcH*=oG#3!75->QW=1x~&w>pELHv(W)GN!hN#)%1#UX-u+lxB)1#BtREF3 z)$kE66Ys3+F53s6(W)6pe*FpFrSwSc*r!kM)djH}>zpC@=ftiJ#>5!hvcj7?S3Lm_ z@7X8`@R^6}{xWp(dNmJM&L2`1`?Ufe7n@T1iQ0rm8T<}?(6Z1^a zhoQJrb|eJOaJetGn1Oh2iI@G`#zqLOdv!J-xd^EwTd_BSf~u+y+431W_5B87ggUg?PidWvsxoN2ni|3YI81A#P0x5Ydm_5l4SH zeBlG$LOgtp&TnrDLr{w||If7DMwm$2jT{$=KrGcCiM-{Uf`G^W%2;`tf@r*<>`H!} zhA=&^sTR|lj+jfI(CIu|jL`j`d!|tH95MQ75XSTRIijppp5Ed17UBL$-=8wkg7~2b z%=roIK(NUnhqPw85&1*T3!!8E2#|6IOhWbx!k2{h%1)R>G}!OzzlTjB)Pu!%S~RB- zB5bSh%(Xd0GwQOPf#d?h5Afm#rCA0UG?Gr{Hkb0s}efJYn=xgo!MF)p}Hg%tWr{ZCt~BWFg-t|E^-* z=OWXolQh*zIb{8vZn)iL1!N6|9=aB$h-^M`cv69+fmAQXwvSLWkg$|ZC*7AC$S^VB zGp!%C$mI+b)R|un$l7kwff~gTd671>8?Wbx%ywblGN-QfFnalc9 zC=+xG*>qzjWdCXy(rN45J@iB(lF!pTzp^zC8N+y`mJ*+jtckWb4?A0kysazyJy4?< zX>_NoH6pbFDf#uBpSSW$aX4Zi2Su2Tj9g{gnT>~1ARgGjQp&4;n#J}XJo44J@E^gpOG8Y%AySp zqe!VJ$)u&v-;gQ_`a@#ODWuPbOFIAT=8>B!N1s767m%sQdf@w%6{P0t%{0+3Tgd6t zVnbN@1EiWAV862V0I76TGW*eW9#q%%tcCYyUX;`j8M3s(@%aDE7~K zk9i0LRrfMJ*_jPN{didNVx$>{!cfvNRi$W@5AkNnKSd1cgv~i(gcBa6idoskFbJsf zOxus+EF!A=81tj74F$FE+c7b7o`P~riGw~5rlL4vEk#}HG?X&tX9Y={i5fzeCW4+b zQ9p!d-fl41s6*|M-P(hys8Nh1Az0-SYVpsPtxKwxP#Ug`dl^_=6wvgH1n;a73M=AQ zu=LLemCioCcMxfes&35cK0kE@b?#qOq)MeV$~0CIeR2v8Q4MD)It0Q@+Lm%oZ&9?GUW7YeO+g+6??yu6Ay7O;P z)@u|)b@w+YZDqz$H$)RE>$r9kMZFp2Ll)lt@v;R)Uff(so@qrf0GiFp*V<7UeEg41 zbUIO4Ya5C#KL${>i3C-dgb`GYackji>I6#dkS=`w(l^xhz7)Fc)(_MLnP-nf0IR4+ zhC{sUyx*u{xnr}2zkZ{x-n$kkt+bC?id0LYN*tiz~P7_7PR}TGL$Ph)FhH5uFvlT<1OR@C0UVaWeS&`^!^yEDH zc)J8ZB=!Ot@xOc7@#phhOtdvd z&;3|C3;pf&7e#Xv2YuS{c^$?<6)hh8^@htk4YWuLymd%S3;joH`a^k28PM{kjfpZVNyL5Co$ff7et(b++D zOF8Gf(BMl}f!}L=(4p7*cY_Cg(4r4BjUGR_|FVW%oXzI=U8Z@lL?EZ#b z9s2gMbCZj!uhCS|j34jA>e2l*9+kn8Z_u9OjUC(7ZD`{i5qT`M3vHF-tuihC0Ug>> z5oJHxi+1#GByJw`qYLKCZT)4xp+9~4*!bgM8ojV7-{PIIfL7=``gotTj6UvMigP@) zfxc6{fUCI6gF!)@Z_7+_i)UR7@cy~+2lnFHe)STjA=<)0+Qkq9l77)%>}`oLs%NUzms?@{`Z6O# z&~}(_j5&$u%gz}2^6QT>3S2Sgx(nkf-F+}`tIxeB*~VZ*_nLv$!uK(>#cRa#_a0!B z-&lgq>?B~MZ*B_YDkNdTJr<|M<&!bj!f}jaWhodGY2?zVMk;0#X8Lgxn}%@}^sJpy zO~c%|$TA)LoQJ77sLZHMD8$Tunm(e`QHnX+HKx3NwH#x1C|i+Et;EP_z6@(Oe1ynXUH?rY359#Hyb^;^vQK^w)>Ev=Z9z@hw8a_tycVi%g6{SH%?W793~ z+lld6&+gPX{T>s-@klBvevf%|Rq$H6=|@b6MW%0Ka33aK+&!!Kc^_urd9v<@m!B{q z7Ry!UJ;Ru!fs%bSNe(7#`U{~CI2wd&F^mY z2=d783mh@b^Wl>%k#5u9zQQj%D^R_Yj1-c^`4&8Fdn+PaW>Ka3kaSX(f3?A1Z}itu<#4pD+vD>e z$yC8RijWO%I(+e0Gc^t0g)5&gTzZ_M5^@t?5XXY?wlM08@=4o7_?+ZmudP z>z_|rH0D#5JxeX1Gz4kMA_HIIE*ojfzJ#cr_z|ot`%EOXL-U@#?Bc`dW41navd-mS z(=8kwWjhq42OMH=%FbE*=*wvGlfCy%Z1Rt6fGpF?lx1%cD0`_ztZ>~nSXSx=KEK21 zmMrgEcb+c(a9NE*Le^Dv5wc63dLN>a9?NEYmOzjSb7gaz&v>!%Ph}MYQeM{=mCL?q zxolYd;Dv0Rm&L}{=9jYgZE@F3@p@Sz1FZ1$$s5_EIlkDd^}VvThix5l90z2*-lMJa z97km{6Y1-91!J-SPY*_lt0rU@tH$El1hTG(SqB$cQnR_Yu>B_t; z0KdCj^>jt{zHF6Qo7OK`IghxA0o9(LoU`6aM4pWlb?`w0iNU(UTCY!PmsR&(l!s;Twl04sF4)e2TYITRkw? zz0Py+-=(rx!;%L_V9f;Vkh6Jmb`k>%joCh^dM1y}LSFk@yP=469-dhtaWt^k`Zlda z`L(bW5rGg_OI>VBuh?6yL0#;A$Mvz&8`jt}pK?+if7xIY?vAv3NLJu-fZHxYY7TPLY2G!a`rdQ2k0J`LMj z|7=0pDGggv7M(4&m5z-a7;@QBd5o>5`K;kz6=0o5Z%@}&KF7wD+C5nvd4;V!tQKhE z)qw@~1#$!zJFrd3*wvThPHb$Ut;x@#4_MF<*|ZA^AF!v z`EkO%_got83*rRzMPZxR<2b9@vGE{)FwS~q$Bwt>B<|QvPXBVsSse6oL(c(D0{5II zbX9gv0;lxgC4n7r0cV_$_pz%<1}A5u@}B<|0QcjQr}BIU2q(R7WMgCq#ywQh1y}un z;GX*)UAo`|#fjW+ZE8RxaFWX+tuopO+}Kf6KBW(Z%e}1}pHPIx!9@)$s-B^7vGn^l z-Tz^6mpoaKG0#aj<4x;=LR}_KB!0Ancb?m!3k@>H71Il;@g zzk1*>(s5&)>Ee40z56D(+9vRc3-`@&l^bEEzepB1C6zelnKu@=jMn2hf_7JN7SnMN zF~1yfy+}R!?zS85=lI|4 zx#hvjd=Y`OINWDe;S`PYyu2JQ6#Ni(9b2J(syYd$WBY4FEiVIC8-3I4TWJ>VYOHAb zPWNNn5k9zP+iX7030?0*k|@AEOOpBk*Dl5te7jOWqL$+-JVu1}H!5-4MS|rCqK!DG z-)wRrqY0Nc{6o_=un9NyBw6a=)izu}fY8+f-|929pe)86Ao@_6&SGKn(^D4e~|GMK`l(8GWYn`xX5@bRhns5xA_ z!#)qXZ4O6nLDoD8{fXnBeDl_6ZUOiECj*{+aS7KK;txDlyN0#c$Ask*0mn$>q zM!<6_QPX>cMEr>nhsc6HB7U)Ry_-DB#Z!-e7REFw;QiT})uS*?ytn$nP#i=LPgOs= z?cHvGcPiMu6RB>9zakR;yI#>0?|$j-m9>0xe6)RX&f^*feAd+@hu1>a@Gfg#QocL5 z;1fEfXNb`r_`9c$9X|r{#6!kEpGo=UhnGmry`lWmA8#UHZWxaZ!J}TT5zh97;q_Fw zV3CjE_)&xNpE@QI@s?i0uiidM!&fZCjD?Nm;G=j|LY&qL@eT1u%=>{wc&@n#ZG>5j zUzwoaoA~@3k9=5@yiP8|yB@niOLwWpr=PCiza#nz59>G^G@($3-;94|?mh7quYCNd z(tvdnzTRE1@k?J5zQ|70zA$bGkD3}>yWu&4XDG!hKB^hTi^n~D9gUpCC!eBUki<{n zZ)82Hkx!bz=b(-pX#SqXU;kTvA0a-6KQi`V|F+o@erM{$aEas!o+tj}@A}1EJm?hm zeEGLS1n%UW)vz`J0!;lY{4hm?;4k89{l)1lLEY%eeaW9^38j?uc&!2;VbI*cWiJ*+ zXg8@$G6y3FUp5tP2-YD8KhiSY{pe^yH}Xvjen6IB$!ZN;7-18>#x~6 z#cC1mWUU7n>lhMN(+=YrZWt2wRlZa`P%t9sKrwv9`9_4JdT$p`y)_}ko)mZ*K5jx# zh?sC$`)y7jzi9fYb;pXZ+H_&o=erf**u}5Aogb|UtsZKY;*$=9)GK9xG-)S7jP~mI zw#Icr2gq6BL$D7aAbLPD?WqrefMo2F)guVY?hCNmI}wD9_5Q=kplCv$@w@!bm9Yc? z=ey6Y%*PSBr%t|>k4z+-QJ_5@70)2lT&b@oDrOUGE*eTj?PL?0wM+81f8`O%?7w<> z=adp&H@y>cUalYjBo?2Y$*UqzdLz{_wKW8e)}b)0v)cAzcKOjNTijA>D+NSy_V}z8?tM>3fZ6 z;Q_*^Q*FA*{m+Cah4(KQLVgeex4s1PoLnFz%%{hS?W_)$FLjBgVzx1awmo$`;Ms7chtxg8J+zFlN)=m-$UV&_?NvBJavI6E-% z;%TCU5v}%+s08sdLU*M4&pG1XljO>q4`qm6JS7_&Za|`{ld16J1u$`%+T34QiX=Mx z`b?TSg(1GrG`fH|LLmO_vI!-okcrc;&I%smP>FCXrX2d6O3dul{`7r;NxXAC;(qQi z7E#6ifD=&8A^vTshQHI0Bi1>x!`5q2aB)S$OAC;=86KB>g z?dxCDBzl~J+YP2)BHBBAz5VZr5%G{UjxqP%i1^^f2g?IdQ=-0fN%;WIl;|q{F6ioQ zQ)0p3onhs5Q{tOAZhzc=3PI1^!X z4dxr_bz=L{?}H87>%=qX(+r|5U5ICduT?hv_9R+6KD@UU6+)C6U3vn}4k4x$k4_yd zg%Rt)Ca|vZaN;v1BYz_FHu2Go2-9%$2;%Ick29CUV~7J2@3mar;)rqL`|Y^KIHD)F z_TPYiB2g3fV)W0q3?hF$#>>DZix_gZ2kOemCbkYnNdEc zooBI`7_IwY_Iq|4v99+7rA@kn=&#Z1oX-D&*xvVlaYV&Oq9q&RHaqc=c=Ze|Ka=^H zSX`vusI@apT&g=c80I)f%!b7U4;uU;E`Lrvc06yD*fV1y!^~PE)>?Ymls2vr&zik3 zQVm}xa&v2&WB>7x&MMZxy;t~2(Ft9KxC~KJ`Lq}Do8ei~lP6(yJtGpN{2sSo=XIn= zH&S@K&lf{T_|O!M=2uXXM+TTCUV$c^Y0ey_B@jrr+&BG&hX|xQhdyvfiZoKDSa7wa zI+L_J+;HB|j!A+?t?xjc6-eKG$7SABE0MyImYeH)lu6_2mNn!971B>OjFkH#$Kbe{Imz#c+Cs79RT3a1jjFHQk3ZQ$nq8dLcQI`zsh?~#BIoy#QbN9bGi>T5oj#pXr&rNOiY)j#H*fu! zr0ey`b4Bb6Y3bJuZtCeV(k)?wF{;8O36XiTFu84-G?k%XUfeiCO7R&sV?bs}C?Sb^ z-H{8V?3;MrE{jP;G!?>QO1xlXy9;)`T1xCKija=I5 zfRS}92N#~6LXiW@D1cit81lBXw)Si*oqQ(Ma`^Ob1+tsldhG|6B6*&4a{XJqGPyg; zb)DzED!EOMFMuedPFA$Rv(O3}(~lsuS8|%kffez`5w7iI-L9&H zV*O5XNSwqq(!*Xd$FqM))%r7eg2z(&mCqnK(VMqWeex@rmHBJz>DCn4N~`CK8|Vj_ zd8Rxww{n3zhkQkRpzxR6biupwywetWNL{jVWcfMnhOQ6G{`&BB9*agt7>xaSvsgQ0_aHyZnM%P_8+=zhVE` zn(`{E?}VD1Gesx=q|#eUXUf3kF}Y)L?vy}~kfX7X7bPs~N+b*7P2umml)4(_O_83C zjc$DFL)q>S=vvJ2qjdP(K|%inQdWeXqki^-%97q87^3P{NP5A=2lh1fC??A(+}(bf;pkAmk>wsrTMo762E?d^;-6E@1)2d0B4rftSN$P1?hDFm^nS(^yNXis zXs3uOq~6wl-A-xy;rCHns*AE^DHE%>*hLAF|I1(b zyNB}fos7t+K_5l=`W;MTavvoG-G0i&ZvtAPSH>t!O*R)Ui%e6J+PaP78s{nM zGo_eOj~^8IHJbB5+yZ5^v+9e4zzW4Yr5lI!`Ag|3TRyTt+oqiNA(%ZdJfNh{?W>g4 z9iry$$P6wd@lYj@Jg}`MK5BP7AqNmRBRoH)}hU$7D4j^PIa-UoP&wW534!U+3PBEJjL=YwzZgiET_cxqukbwY=FHM$~f-rI<}+n?P)_cNx}jrz2BtC~_>zH@7x0p`@!U&n^d z7CTaF)xcL$;n%6M?t}55EElSQ?v%~XR1fNY?%ji1k)BjJ?IQ@Cc2BCIk`1RI*o*3V z33<3E#Fy&1Lo#^X>_-*zB?<R*F*W%r@B355MO4u0u(?xq36)aGSd%a zCNlE%MW)elnnx61vYdH>rU(j=3$~V_6^Z7?f>oh3!z)RxhbNG&x+%?n+2+CDKr@=!9C++avKcK(!)|-X z)spsBdk-EiYefq{KNBl4w4zN^#B}&-ThrJl-hY?Sze;=X!>IJ>2YXsSSui;GsuL|& zsDNhxKp|tLxGTg$eFdC9zNzSedqZL;Qs_c%1)7qe?&b>8@ zq5*(E=HIT~r3o)*zdRlsOFO{}9uR$hpH{bx##BChNaOu&G>il#(QXJgrRpo@2 zw6~;g>UqspnvUcAUG73V?fO+4d7HF%G-`&sjlWVa?M+PjfI0XRt>n1x;=t-Bnl$vP ze(v}%?R8?*-}&&bwAB%R6o2U|ZC(V@_&4t_Eo3DT16ut@D=(8iD);*zZ7n_SxREp; zeW2-_Lt>&Z9WfiASP4H(f9&U0#Q#Q&epB0#%TscW{$pVk=uWOZuBGc3`cARv4CFqU&f6asp>u*x zUwf1h-t&!3|Jbip^ztK@4op3M%;SJd4_rnyQj*o^odpp@F~>`E)t5slDP=nJj$hmB z->>S@yPSp73ZQ!QAdH>J^qd9#fe9()&T&gR`EJ5Q72TFTsQbQadc%>Pz*kfSGjpQf z{<)fVBin_}bN76&c&R6y29FV;9eC1HZG4j+I{DITb_Wz)CT@2%3HkLlZcr9Q`aa_Q4AZ$E&i%h&> z&-AaPIQG<|LAnAjVe$YyN*6Hu9U7H4N|$^H>H8%8jc%~M^3#uZp3Z8@7b-vXgPwEt zL2qEm4?4&CDv68#Nq3$e7Dg|u(vK$5*1hyM=-wx?u2q3I>Ae)QcSmr$bPqmU>CaLg zhI#WGQ9gUWmD^X;G2Hx7gTCc=_55sdic~uw%SF_)&E@?7;TbARj9}vYi-u*J$UBn%o&6{}qXN zAwNb3k|0-t3uUByZ=F#Ji)46i-j)#^i(@Q-#yGMpBUW#{#kg>;ijTs#%?KS#6dj!2W*l0h{IlHHW&mrhdA&0F z$N0Ccj6om-nJOS`UHe|bq{=B2tLesz)zrqBXpho_v$l&{|PzbCK2oECqg_V14p z^I_y&#EEZeOfETVHe6es$$oR1|Ea1L(`xJDEWT2kxmF|*Y2T^C9DR3j@8_-_bEFk{ z@6mt}vrb3)2hUYgW+0(??f#@WbE0u=HLA~=Y4P_J-xBjG6Z&FkIbFz}c{fxhW?s^b zdG&?L?T&5_ro&^g>lRmhnI!k-o9tv?W-8pQ-QL`v*#Z51O>`xI`Bi2cS=n}rX~KiN zx~qAc87qO?on4Dy0wr}D9M0ZhhU^7h%()oJw0}F?d-7Wp^9Ju$)`hZY=CfCEQEOiJ zm|FjPm+u2(m=RFy!kg{~On1reb`~oSn5BnOMkA!+n1iidle_p-rtyWg%yNZP<~z!t zzxa0WH%8IIHer4y|u>f^Uc3gCd!R{Mo(r7=$ z{#6t6z=rQaRa7%mi$QpvCH#(gS!4&*f$d}tIu}+Cz3pTMoCJLEYyQBrJ8QtdH`T|C zxhniukJ!(2+&|#590!@9u}eQ6IZiT}s;?y*GbWkYqec+b&IP7C>D0)){vxv>?1G;8 z@+MPso@LMI-(gPFErx?scbWD|bl0e>hgjm<3BQ}L0<7tiUZ*WI1zEOFjjPkdPq8{) zs??xfi?Z&vXy~?l7iTrPC7)NaIM0H9Bwp_>l47-x=cY{~q**~XtU7LF0$FPwvihNO zVAh$K?~i}tAuOeb-EYnjA*}Dmoz{WyAC_f&zw@>ljBa@JI2x$ir>JZMs3H6C&~VU?=E+QVP-qqJ+X zlIQneZQhqy!vwzT=Nm7vT-tVbAmTbKy-)+IC6YcXSy3#o;+7Gszfj$w_mwd#tgvRr z`H&^6DXgv>bJ~{GWGtB%*>{x%iMjq+q|TX@$u02J{^QC*Sz?r%^l!2*3Kl)~N%myX zKF02FE_<``=T#xqszI#Dpg%QNwPIL~DQ}XGO+IA#_)fo5Ye;8_FRI)INoTO)Kc5g^ zsLo~igXRi?r1M#E;`Q=~umaX@>hhm_(sNc&e~w~EdnGI5hv@VMxsLTgzA-=Bw}Hjh zx-`yP)WEWkaWB21*vb0K#J5O9^sv5k?EE3ve_-VLc3$QhL{VVma2(mxMoK6#(6l4#P(ErorPqOtd z-4pZHIL-dZ#((5Nh_NUCo-H5Rlw@xmnfH4bD8(*OBYYL*muBZIUPpz$xWN8sFIb`V z2gLT5G7!7di(serD>qJ?W7$=-r}kQ&M0SB4d2!pC%C3*jZp!Rtu!klOs68AO`)oeX z(24__ZT8t@&{IpEJ;<~;ulY`m?RML2=krlbHoit#NG(pA9V=y+R)k`#QC&26eE;X>Wnv=0wi zLapxX>+Ey19$p`|1TkByV={oPui%vDFc!$>pFAXT-6M#-w{qpJkJl~sq8P)n{wdBSj{^|} z4KYr3XpdVteAzjPLDB4jup zZ`N4VOMy5-haH@-)iBO+4W;7S*WjGKPd>(m@<>kPXN6JlIFjR#aP(aEIEqti4VFU) z;yLYBh46`JJV*G#V5~ohzzGQ}hMS(Hb8bv%X0;GmoEIUvHL1HwoS#)=@6&%?0rP9Z-;oDYGbsOa9ULdBo5@tA^d%TIy zZDu&{71@s!Tjn_wi>!V#&;qC9v}NOL*(wL(GGddmyTS2l!hGw@-sb!>JhgQKw#(T^ zwwAT7?Qym*4S8-gALbrb8w|Dz;p4uFw(C-a3UPgb(Dgln7Ww{T(m{;+kvTOsO`8b8(TF`KnI{uHm28atZu6u2jF=ySh|7cM$T{?x-n^dpeI@ zv_NBVH`PxYHuNUFxv#I3 zA*S4Oc^ZrAji%h2r-iznEt_!*&=Hb|UuN8nsmb#WGM3zr`st23lq=k5@-$-q-4$-v zJ+{IRQ!B27O>T1`z=m60GrNF_cIF;MryRQ$)QcN^Bg^kjk}r2iNL|q+ zE|B|CdNH@iGnl*oV49%e9?8X2KfkUecbEG)8QW;p8_Vr{FZbm2{RiAPD?Fd(_L8|Z z4u@c3i|O2Zt3wj(m`7YyZD8rq;T*0^*)wS6w><7wySrQDvL{?)ySslkO-i`SD+J>d z<}>bG*^AEl!xdbUfOJl$aRnFDBK0H}`I5_8zuwoL`i7gfMgyerws3#iv15siE!?lp zTJ)m*Hg1#akNil7POev{n}1o#828a(Oi zr@rce2+O^(IK-J56p?d{y)v|PR#a}SJ1SvTQe5s3aNTkuMqCc))8$ehcR?T4IdziQFO{bX0UGLG+w|G~#{&43T)L61D-Zsq0vsCV&_Q!bEc7ys=emw-z4DmWA< zcWtY5!*3);?y=WB-A~73YNp2~f&ZcZdqE96E^k~*$BRLX4)C@1Do8s%t>5Kc0A z)+6_BTRr~khXFYa(iV88__LhWcfqmbfMGeKZ?o8*vafOjw&!FmUwo5;%GlB?mZs!h zo*1vudNwVmtYP33)iytgfv7%(2QwY4J`f*tv{sCr0#Z^mAs zJcdV}oYv7bditom=5Zmr`irOJd2aTcZp)RBZ;8qrd}|~r-yVm=mQ6^@hnZZl1~|jz zcVw{#5Th7S(4_-e$AFs#r^)lM|gxRP%QQBllb-mMbv`chm{$|}vl)fck z(LLq^Dv~r~`QC3s$*#P1$palocPV1H%c>KV1&a*QthJa5sO^yU zhKl7pwM%t~JWhSg%B0OGV(SF{pqh^get+dtznz~3rnk7}$7%k7d&a#)&Q9y$Q`270 znoSWF+NYFr;~fKs%uQxg*ji#?)vZGVmn|{st#tlLv%$ysqbldD4q?Syv0S&6!?@)5 z=9?jL_E_nRjb+-%X}nA^%h8T?#ZV+)mw5NOHAx2J^7G(#)#$l&IHzU$REtp&|>xYS~~Lw!GDR-q~0TkZ*q*Np4< zg$6{R4Ih_%D`kaI8dGF@Nnh{4MwAnX>vpe6?0KQHeW=e}Lq)Pumo> z%cDI72j+3R^R=+z{vTM+HrjG9VF{~7>JUXKAy_MTA(9A)fb1n5cI;je z=+gZmYFr};p0Z*6m?{GjhJGuY>R&-mJX=iMe>)6&uhyPIJK*5Z!2jOKDFTI7s`Da6 z33}{{L#Eg?(0eO;bmo-`I1Kp*jCQJm(1KZJp{*L^YVkha9#w~#gZ;<8FtkDQa8Bb? z;x0J+G-3ZWIRhwTU*3A)^ImARCt<@gMi8oIe*KHSF(kb*dZ`=+aNH)p^u-Gk$ak+` z+YknLZ#UU?u-F`q@L8dOiI(7N-n!+vh&6;Mc2h)6?Vu^+b9i8aJ+uqI{Ao~P4~%o= z>mNUKfF8kYHStjeezg!RzE`k5oGrJn~OMz37EK!qQ z4%*||l?6k5sP{i`zu2(?&YwAd*6~g4YEH|xqq33ctan?IG2BQ> zZ|-+7j@(F2S;k(?$)b|{<<5iDDk@nNPri`nq(I6kVrhcO3gqHoQ107nswA{G^@@hs zPNEv0%=sligY@k=&AeQrLF^u0GExc9A`VygZ-~#+A{MJ7{FmGH2n6v54qVkEk}0L> zRgL-t8uAsl-`Ptzrj_nbEfJYxiB^={BV@kH;bexE895htqT9gLg0yvgq0v&U$XLol zgSTx~#PM66O>FTI;=&A%Khb@JENGpq*Vh=b%wH{=$^;U1^b5HVqR)0y0?oaL&${0thMiS<((5R#rkt9||Iyb0@O)M*4w%sO3y^)Nh#~3G0 z{YysP^xw8F?jlWpj(jX)d?Q1bn0piwM#zP1lLqse5weE)d+$QXD9Lpl<@z;@6XxM` zu5}2= EAGv0exBvhE literal 0 HcmV?d00001 diff --git a/velox/external/theta/tests/test_sketch_files/theta_compact_estimation_from_java_v2.sk b/velox/external/theta/tests/test_sketch_files/theta_compact_estimation_from_java_v2.sk new file mode 100644 index 0000000000000000000000000000000000000000..e4952977a4ab71fa3f8b4ed612340eebc1c13953 GIT binary patch literal 34760 zcmWKPXFL@S14i$5uY0e%_mWa1BPwKMkCY^&j3P6ekcjL(BVD$jz~&vYSiya;%Z?*@$wkppBIUij{dQUWZ*6x5zZ zD*@i}W~glJDFd2j_HB-GQ~@#kMYAS7YJfrI?0BcN%Yeq#ZtbEdeE_8Z{_CRFO~9Qp ze;j4^Cg4M|nQQ))AZWp+ zQnS_#kfJ$tC;@%|kdi6@SBO3UeD@R2`hVafUSj$l2;uExAU(Fg59KpMMMB_5~|> zb(=20J8S!hsm4Kos9v^5K+_<=Qx*|ezwr@ZaCNTc@z)_h#>Umy=W$;Etv6F*HPyZX zj(5vU&zv0tT(<_wZ_G{tR6_2nC3sE)q@luw_vB{)-5!l~LK-swq5YH9jkdFZ5!iC+ znfW<@9jHuyjr%*m^1ELItzi*xs~}Cu=I16rtI2oWS78V6NmbNt680Mqw!I=yytNPT zWSlxLu?PY}%KifWy9WV&?=r+cZ-)WHCj&@FcrPXjZ`y936*hyoKw z+%&sI&jPEAmMj}CiUafefiP{71dyKjnjJ?u2OLqeRz77a35>4e`Q!0L3V3wDbN7so zJP`EaA^gsUA`qE&RQ|~GGBCO1>{o{3Rbbg2Qod$_Hc;s&O6=O69c&|beZ+A7-z=iLaOV6`SFS{@k*46okP7=T0p_ockT2`6HKVtOz99{ME! zSnuuMQ8AUXlVh^xzjQ-x3U0O+myBP zH?s)HfBE78H?0!*xZ>e-ZD1=eIT!_YUXEH? zcrpgOd3*IzV$XlTvS$cBCgrk88Mjb6VTS@GV^g~L+K8aAN}uWL z<|NQ@h00L9UJ{5utUlr16*9jk%D|62!YASlX`U`O>01NDAXm-bkS1btesE~iyJ1to1oc=1_0 z137H%hv`Q@1Kqws-?j6|@{&)jzDB)fy4$0 ztmZE-Kp!o80SdpQ{(b0o2WU*%!k`h_1-fI#Jxm?z0xil~b)V;a1vUS7 zQ~2?tF_5Fx+SRD8F;F^wUPS)y0;tg*VG+k%1(jS^KE@u}1nIwsJ|nEY3mUwsY|V22 z1Nv({KV5Qc50rJXdRKYq08|Q}6AC{32SV^(ddZrDfkmgeGy{yez+#`qTi< z4p>hbSa8QiOUHsAT>AVI?WOZ6aC@!>t>l#;n6>U~MwS%=+hQxA1L9}F-%2JPV3^|I zj`vXfthNLgpNF$a^pXVcKJ|*fZgLR}n3U+aj#dXp(aeTwb#=giY~TAOxq9Fmi+o&R zBe%f+_=9Q{>Ob^$1+g;v{^ve+16JE}EwmdxIsfKi5i2^8t$vJ$(~H4gpv2AHOs_8VasD zTu4FfM}b>^EYUhUx}aXSQ^BXfCY_hl)4)B8YeD6CFTmqB{6!|bUxDjvBM#j| zi@_kXR~jPI#o%MCp!S{XM94syWgG?Bx1XC}PqF;Th2g5ogpr>RS!8$sM z(DKWT;Ah!3=ezzif+MWNWxMv8z?V%fB)JrFKLfTV!0%T@k{d{q;Klr# z*XH#=Z`TjN<4eT#2K*8D&$rite9B8B^=K|T<=<>)tSz+;H|am!bxs=g5EC+VB@15Y4J1&uaG$PkF)_CW-+C=BA( z3TQ9zi-w&3cQ{lBkAd72brk%%8V8Z+9NJ->O@@5`D&KK)GXs*L{BKs@KO0he%4y}o z-5kjN&3cg?yE@41x52FbmG=;B!qfiK&^8ELX8M{xxdTF+ab@f7_CS*BI5OeOy%3Ri zMb3*&gOEq+UmMJR4MMa}o{+?Fh9I>TjwxTIMImm6sa&>RPcgSsf{&Vs*^AIC`h4Ws%n-I~xz2bu5KaiAGm)+|N(kT%Dm2Qk1q#&M^iIkC^)8RiK?|FMPK4!(*(?0)AhR! zjVa*K$f6oTo4JN8(Q)Qb?soU>w>uWl0K2{HXUf)4-Bp?_q?d`3Yib(n=%Mw zh`)l8U{uLHi9+ayFCWt&(nZiSu6iM%XN#eCmv-#-giGif^8It5^fQcfq`cOR^cdJK!}G=+^^P*A57N3F(IxiA$>MgIGD(OcKc#K5f&vp`TVp#1;(yAyE9lohaJ9-zM1023ri6I z481=(4tv>dy@+~r3YPzvFJ7is2p0aa{ZQFm7`E4Ow?E^HFl^d?THZh4ENtBMMPkS= z37Asm!|T*>DVTNSTfQnYC0HBo%&_RDIxHZ#%YC^~7lwWC<&)0S8?YNDeM$on#<1R3 zr!aSscVKC^*3Blr+<`5-fMT!U&0&qo%nLWV>|nU<=6_2!9AGg0it-^j7npUreUrBa<0~`8#c*B6F8a7()(f;;Y9nA5}FKyStdYF2_tGI2i9@w}7dUa8! zA65tu{&&IV6Ksb4?~ba#R~YV4cFm??0_L9Me=HNU2m?RGNS@JJh8_JYXp^$tg#Fb= zFrBS_!LI+DD$VZPf_2$5e&g%@z__#Xej2F)xnwt5#iZ|{xEyJx>NO;=T#x2VXSyT^ zT-2XElu`)w zQ!e7y64Ue7Y`G$LC}?fI`&{G`iAqWB54iRvua-#*JmQKzp1<->-;Yb-#4B$8yMA1q zxm|GqN4G8%Q>6t%gL(|-t{lJs=Uuj zCIu95rFcIa&=-8o6)ral&atZCGJ3Do-^^9TCG_))%+}pHE?H4;DmSK{%Z>);Y2U2p zN(x4|N3S(-^@lY*>mG0AdjHINGS{Jt3*soa&vozY^P57# z3tY9Y2evD(tZ}VGJipx3w9XZi6pb}A+2op9-dH{m-sa+e8xuEYy2GXZ>8ZtB90;!S zq3dnM3e%8s1%(ei_=8l)g~K_n(LWvhdEnHS%GKqciSWfC>yY0!DR3=st!gm^ zI{dATgOjHR6Yll7K32k=4=(p83g+h`1UE4RpVX=sg6{-2?$!2+!jZS`x+y#rgYR_g z-AEj}0B<lt{w>5CaYtPhF0zrHEIFVK&++<+i1?-nr*>o^PTbIfm-ufhA;c+zfQS$ z7fvWae9&vW3x^e3jd|hC-~!}9{}&NvaLCkZctO%VxLxATzlgW@;7{aUx!X?Kz^C>< z4u=q&;C?YQ+fXkjxcbHDSu>Oi{A{d-6mI$fyb$vcrK95sS1l#ja20#P2d*^e=h%6} zcS^nvzdQ7XUvyvP|G66g|7*K;1jkdL3iyDExC5?oT@kT@;7)%tY! z0|iQ-NACE%A<1lQQBN_$aybr0qTTH*V^x0qs|~ix+CS6 zP0t|yT<~tuZ52bfnkY)%@s&g<+ixxrrzH{M=g;bVme53e*peZilGQ>yi1+4?Gu1}S zo89Y4t81QF*9B*U#NF*5;@4dewc%BUOBdV_1rI)sm9~2#oc}?OKAQw1 zCXcJ*MXZAn5k9{>#!7<`uUftd-}xJgpe;nN{82&f-z^eZbsR33Y$ZHRb{aEdK7gR);Ea*Yxx{o197zmKNhG|?&$z5RA? zhP&P(P=CKI4_>ZCcpjvG_@-Hh_%N%kO26HV2y%51W=nJ+Z1x?mSl9I+MACalqs;pe zd+#gHm!=LPhF2U<91Hu1coiw3_hw=gAz3Z;;&R6~#EUwW7ZJv@h!@3{_{+!E5Km_w z6j|ebA+CDsENB9^5mlDYLEP%w2DQ=Pg za{sfG3@H4L-YPyHoL&w?m-{w*YSt{}Z3<+^y5+3i2d%xqqaX30!cw!AqH;9m_q7A*x0JsCkqv{dg3kZoLOxx)XhH;FkU=emCL`7ur2a#> zl!S3C5=>V)KHiH*(s$LHj#pBUt!M53iqz1MR|=2%cz+5a`xE10O5Y11Pahb#Kz@lJ zsb$mg{AuFI3U(w@m0tt-tKg34>u?PuwCJNI;hH|OyRy|>xyl@Arrm#D$uPMLOfeFFYGq8wvmEK) zm!=rERfTMxdL6m?y&maTy3y1LY(V-7D7G7!H6YDM>qS0XjmVQt!KWXDHX~6hYZdR` zw;?G93NzM>cH~=LxAO((yO8cmpI-XybsLzNQG(_^>Orp8 zcwNTZ_aa3V&zK$c4In+PC7_`$W5}Q&-3L8gE6A(3i%~*@zmQG)JRs) zMAZTbm6tZ_D07*O(mjLYalgw!X*UnL5L`~6WVYQ7GClcGortF1fkS>&^3GP~{Fo4G z6U7<{j}b#vNyqxKICy(XwOXMx%CU{h4lc}K?uF;kSr z;qVZjqB#ncx_gdC$O83UPgJYyjU6hYb1lUU zc%Ui{@0@wp`WO|Bv+5MK@I&=ao~@!qJVCW_d5Edd!cfMAJ;E`%VJJZ7{ns*b;ix6< z3hzHAktkP6q};q?B+4)5skhoz94Z{H?~45shvK$zq=ep2L3tJZq^e7&p=3uRujVME zqj=^Ad4k4sP$tILMNG8bpq^w?bi}2rP?=Th^PJDss6(u9W{CQG6z<%Uf3J@Bph^V1 zxcWv0QHPx!8nDbysOM6UYA3!SloB@WO5Wrb)c3fdStrT}YN;*5_Lt)bici7x`e5G( z3gTN?8p`z_>V%9|)rj#7O3=;yosRnq>fD!u?=8LaC^Tgb_%Le$Wq#+QUFGpb)O~{Y zOV#VYQ2z`q@!$LRQK8|cQX3kFD4zbIds*c`^yAG;UX7!?sYKGvr37X(7R0Z{xRf3wS6|aRIW(!z9=u6=gU>E z-~nE=dAi!hCi^6MGAXO{c*j}vojAhm-JpoL%*R2P0hOKrPD{Qa^#!nKdCwU4vNNTnA|SI?>}R-`0fuN zGtv~@u9Y(z%rr+o;XH_?X4#^frm-@r=}zcxpFttAB@fYowc>qyMt%F*b1`Z~=gKSZP1ToI?a&&8vGAC4RiQHkhxSv)x%9jWMX$q}B4 zQVWxPdac)TmTe7OS z@@Ph1Ira-(y3&IFATyq(Q`m-fLMjKZee6PS=@5BH2i<6uSLssi3m?#t0dn$ChXHiG zQVCq7brd~xueZ-beccXauzqz&y$sOxmSDFwX_mnJ>!jNS+6du2$ z)YV)g8joX>i|L&!44$&58fOH$nLO_`rlL=_^7H7oW_T=JI?a>eRPHKy|1?kDPPnji ztr*X9-13P-pd^ofKq)UZR+8s%;OO+k4H+J8O(Q|f9~mC>-w=T%Gi4rlYfI~kcy%7p zRsHuGTiQJL3jW%7_ut@Aoub5de!I<6s}n!rUu?>gMP%t4#NOd~ckH{)x}6n|7k7L4 zT_qcyHwnOW?lEVcKlCy6?>?S9tw~qH2wxuYEE&{=Ua<}0k?wba<(v!QxhJ_2^?W*% z$K74P?QVPokHTDI3(w;m9ys`;<@WFt92dL%#)S%@|m1?HxDvmaI0KwfCm`py6^w= zBhNpmhTh@jPdpXo$eeA+Fi+gYMJD^iD9@G8q4zUO$9c|gXD_WR&GGDK<{Kt@{N?HD zT5^@Gfna7bzs-*>ATW;Vp<)S}7>sqZXi@4M4nuL-t+gf+FxC}q;H-;ej76eBe=_Q~O3c&gIqX2N_J&A5Kjhrr{QuvU!+Vlrni7J9_aqqZ}c`|*)TBYXJ?5qR`XB}jJLsApb{}zgQ zHGVbdLT4BTdl#Mb3mb!Z*jI66>lB0O(zUf$u1Uf)d~(%WZAir^8|hzI-%P`}zR6Nm z=}O0(_?we0p`L^BIU#v1>#z_rs~Nd!4=ll;l^&K3$dzM)#r3Lu$0{)4=RBX=Xx3ve zyLWv*$+cjVU#{2tAiFWD=Jbb+UwSbymSs&(xcf0G+~qA_G>0)S-p+jrfJ|VVjmKJB zppzK6(Xj=yiYZLt{m%onjnkOd=_xsiH)b#vxn=6<1xpxgd)c$g&C8glbLA;!@*9}w zoCfOU&<%_UW^+B_%`Z&ehpB~y;cZMwwF=>;{2xrnb?&OM|MoEGpKDwJ69*W+!(^Ko z-y@9dU57JD0Y{jB4^$|Dcm81n??~(J=mD|gE5eb>F+i;SdEWHL2{0^U{lNKd02g-8 zvkQf|$c=q#dG%xKeI(Yg`s8;>=>59F5wld$asos$zJ z3N|jhSlXqTiUn#7F8UwRu`;LZ=7tZ1vB$^jL5>IKus_Ye`fE*Jz?N^tBd;w;W6d(T zFdz|mtXt^)=dCX;VRNr+-zDKxunWAmul#pP1$&~Y)Uq({GIsd(AwP9N8*7C;|KgvN zA@*#UMPJwrJM46ej?%GQd+gn231QyX4p`ri@cTcepI~z;bZd?#0**>K&ADI!#e{w%y6FmA~2OsOl_V2pdHWUtG zzaE=`I?_h4%i__-Pi4liv-$tblFm+H^(ptaI<8G)pWoJ3XRLq69(-0f`O$D5>zja7 zh^Sb^-c*GhRa-7&Cp7%hR!;uH-v0}$b$h;xZFp2i-jmtGu1EZS{@WRV8%sjsJuiWA z|4CIVZ(2ZcuRkz9x+bA;2tg_A?2GGl3lH`zH`^%V>vt+RWp)Voy^A zx02&H%-jnTsm=d)R?C`Ok07qZAhf0@T?hx(^9<*;xP&thwCasmQO2$QZLxKIsEso` zw=@`7p@TEH%<85K>)|XebyBot3~{$*H69FD-NIdXj;-58nBbg*8m9N`OmH%M=dDof z_Bi)T+16tBU2qd?_HUW}fjIXFun^QM9Oo7CUics-9=B9>a5(CegnNUFtG)Z{84mDJ z{lYXS3umgrMY&JN!CfiR4jif}z+D>@wNGv@!Ud{6&PiP^#Z~6kZS6GI;NmlOjDI`T z;p|oEV+IP%xc+LTo!%!sxWs*uOLgEET%qDx^30`S9Jd1zLADyk`2=kjHRg=srtW$7 z<)(kbB@v!?oaXrtmz0~=)#>vSXY6K9Uxe-9fR(pOFH!#D8ecgZX0X6`kIpzoeH<7M zvFPM~cOHT7G~#j+eJKk z*!!MRi5%WZ_vSvJ{}LYe{E_|8f+C)-iqyL|t&HCdg$j^=XyP4(mUYf2XyL0aye7=N z)5d4TZC6R1(8sevc^hK4Z{fvaR#l#yxrgt&i1bwJzlWbrYm@F+cEG=p>*qTNaKsA) zvCeCsal_Y1eav!+^uX&}`!sB*Fd=vZPAO^qqKo*cy8;2j%*LgGqdy1!7L`Iz&O~gm` zpff51)9|Q=m9YzTIrvLE`Kd}d`S_9Uk?)`8^YQ7As!N$S3h?qa;tjBf5`5sZzsF@V z{_n*%Eppmr_-HzEKUt>?ulmMtZ6>-5?{ysCnfdw+-qGKY=$TcCSK_v)aK2QH@7G17 zx`Z_0kISYm-k58`4}AIb;*h@$ACvD(@JMRI*DvfoHGbcQKWS9rBdXPo|5`Z)HqUIw z7w)?V&ct@$-9G1xL)W_TK*$|MmyQqkk)uq9LH|#9ts8<4zlOizqu?>UzlW#sg_9Ww zsmn8Xkt`JcBY7Twn|7r)iWFhoERAtR(oHL&(^f2p1ARMc|9`PO^|aO$djJw%h*} zCg>DtZWJj?63&)c->_|wA$a{fzOQ~$mVkYmVrOS8PuR;^u$4ZqK&aJEs()RqL^ysR z{XlVAhtN~2)4!X2i_jyPQV73vmtdou|9ZvZF2SpkZoK^NE@7X4FS&~Q9zj+_J@ZJz zi4giW`Jc^WcY^ok5ItJggJ4)K;G0nNkWeGa!rqeiBK(~WG=i3R5e~Paop9_&1P`=; zJW}EjL7_(QXY7_QA=P!NGfmWwAoTU9AHU>BkeC1T?QkN1pepiUvraCSkRY(6(L#zN z2)^sF%3hBnh*y*!+B{1joL{_Lo-6T`@I5-nzzmW?_%${A##kkd0Q2XJ*nY?+@LlYq zlx)5rT)4R(rB<0s*q16y2`_s|xYnhX!V^$Hm?wL}*SIVVGNYi7n7e$od8E zmi6o>Yz+?<`M($>)K&}@JvJVS{yI$oE+m?$2aeQ`?#oskT+ic&XB;p2P4e7O_6xs z=Gyl@31woU02j*Ui6-&lXso8K^<|zVIKBX6`1mtGYl0gMOx|y<8$D z$6j4lGX6nK%cwoMR{VqbEv#!PPj!b_vLmJ?Q@=wzHR?BrDBLBs3UATBckB|EKz}9P z3j84^BnMoWulh@rJv5$*9sNfPxFC*7(gKrW6kV%S~7E&AHbka4oAC*6bc}X5Dl`nzBW2AzL0K*N<6Qs`P-KhvHKPhwb&#QBt z!XyFm&q?|R5z@oS3duUVvm~dEh17~bY0^nw=fd277fIz-$OQKaSrS1>E)H=K*8fL zZjus~PEP$*G9q2pQ13<>S(3yQs!c}PtVj*&o{W;?wj`F?wcaZ~>_~4Q^nz*$chYEK zbl}%QPm;vaPl$ZPL(=l9=e1W)d`SbRQr_Jj^dpU`nQw%NN0Ja%a60J^B1s{D3H#6b zQ6$Bv@1195lSuz%g$ZVEr;K@m&dWFGz(OQ<27-Ii#j1ez$*J$Ro9h?;f{@=aX9aU9shD`6S~>oZv^1 zB9bY#9>{{dCPlWsOE?q#hNNs-^UWf;lEnYvX^Z%N8>!U(V~+K17sk=M-t|xjg!Z-uOxEuf1S2_W2BlojbSf=i>n+8LRPiXKIhJhBL5z_dH%`|4!PHvbNWZjF)|mtKV9_Jak4r2@jglgDtYc8;Hf~b23baH6u0c7MXnSlteA&sk+({IU(sl}K^9OL=Tpcw zAiMb3WsZ(nli@vOVjMw#SiXEjw7CS$bfl}6@G>5DJyxAlSPovJN+J!$!CPZ z9A_Srqx+iAqZES3FjrsEwz?p)$BL-a1;0=---Uu%NM9KFQIx={a#9l6?Ob)V(9>k{ z;ZDlr@}D%a%OgG?U|u>ob};#8uVoflag`c;StOh6tm$A#4a_Fbyj$#+RLCV;n~fjH zM7<=hWMiCVE)%30foAN$E$v$7KSjUjTzXHze$o1^5=S>ABJ ztWok`wfXDhiE;9}hlcm4?G*VOpX?%We1V+i7k*K{ahd$dG+QY;{Rep*K$G2e-XJ%O z6yM!m-XOnTshORS`APPc67Uu+-zAGnroVG7Iv{_)_BEGV@Q@sdgQ>hKJ|ZJ2*gbI% z07aRv&C!k*M!D#eP?A{!qqI0({qwh*i-IzU)E?D`Q(~jcB0qk{QACYyPuq+UD7T(F z2C{u96ev2@{jiHe0rrK6tt<0U+;uipkhjiIz<=kVXa{jhsD_orOO`Yxsj`|Ij+3SA zwRPIn3&~QHGVr zZ$~i=hyKh#xlk4sPbov@o=`SlbzSOo2&UwaZkBmsQYZnWJ9X;?DHQ#0I8z}`8YNfO zfAwTjE+rsONYv0IkCG#m9k4o`M`0?A!0ypXD9Bd|PHt9}l$CnDbG%Vil**34j_*}f zl+SD5-*3*fHqi-B^j!~pLv58lv#wdoBA+2quvlNSAcJiHX3zWdC4kfCYixl_N3)MHpmnbiI z5asFhOB9~fZI4qId8ii8!8aH1cLal%6zY{bRhH8qOvQRo4O|2se!J>J72#RnyMI5J8u((3zAqI@ zwaQW!c65)R_R)o9VTTb^ImLQ0;pjMO>7a&(XK(_wY1MX2yZ#w98t5W^uPcLUe9i1j z-|ZJvdKAyL8zPq~;ok^#Fe{+y$SLKXtS+X$j@W#bSWrU!m>Kw=^t;zo_hDSmCGRro zb-Oq&M$a3nEiC@@Q%*Uxm~<#x+*d)JzAW>fmUT6?Kwx3Zhp&dJu5r|u8d*cdEXUy| z&el@Bv`!wAj;f^+i~nujPOPJz&`g;lO}0?$aV3>W^{v!%Zm2gX+6U^Yvqf9E%V+A4 zo+phq@`Y-hUoZNsbDVnRzUZLXEw|-15{iIe+ItAD_ZBu28KfK9v-K9!6 zNuF0n?olOIpIp!6KcHrRDxv~K?Uhrti1G_NZo zj}10NX^%vg0_;-G(&*#=5%-Rrrx72g9@HT((#GR99=adO(!9oVvu&I&(GV%Pk5!gR zw0BG0VPeI~v@oiQWkH-O&7$_9=x)0nZD>fgZ+%mb)+y=tCTd=vHXtcm^)UT5EjQ~_ zJl_>#+Uu=?9|i4qY3h~tLyQ*8X}9zwiH~8Hv>4yT7skw zTA+vZ?OL!OO?cKGnzkKCt8q=L)*lL?HES59KgkWFC8b9mRep(}z0cUB{1l6(T>xsl z5j2RW;b%Su(~T2p+GZyS=YdJI+Wo!Qjo>sI{k`lzk;XJyx{<-}MzJiKw@6)lepN2b z@7!ka3-uz}$FRzS#_}Rs!?j0u=FgPSS`x~Jvol`PUVwt1f+NakuIx&*PsSVCWQfl4 zv5X4Ztl?+1Q?b=F5%Yl7lj=3J9^R=}Cw<@3ylehN$KCCr<(?`}ghhRzA=0_{@Z+EU zzt{M2lEnz^y=B(}d(tSat4A%Jb#0UeIkx09v@}8MJ8_J=(0GzI2T09~od1tD@_6c& z9d3puqy@psyzDPFl#rz-}e!)k@*({DrQT{p}k z8ju9Khx_rAiedu&JofN0uAEB0#@c`-xY6lO6DRYGAPlql*}j{+dCGKVtGBcES5>-7Y`}x+3N5SV;?BhS2`73# zdQzDt>`FIJn~_Vt=}NC`Q7rek`iO2M)z`K@3)xfMD z_M?X=;vVHZdO|-I)bb*2Gl>38VyY8d7);L!o40aw38M$U-F|TGaU5OGo0)Oj<0)Ne za+WsRokHigiW%0qlu0kGZQK5O@g;pZfD2?<^os7d??6eCDxurvGE8!0%IV)0r>;;* zRrFJHoIh%mxAYO0f)filHFU|z>9H!8IyzkmD5hFlPd^`<_V1}vBfUbxrQ&CF6Mf(s zjA#?tOUDm|Q|<}((Z8pA^)qh_&^L}N2`Rt&MAw`@-ltGMN-r^*wc=!s(Q692TnV?v z>CVsVq<-&D(N~+4Jl6C7qrY#pgPa8{(62pB#dY-jpohEP{SB(xq5}foQiYEHrmMZ2 zo4FwSmwx$O%9-2O{?U);gq$gl12bf=IX$>s1ZG?jimR8HfG{8{K2q2Hkqm5=QwBH_ z$pHKKUf?C882iVsC==db7>jQiYspn(Vp>+{PpU;6@1&e&1oJy5H_%`s1z3NHpaUavRgzbA~fN_3gHYXAe`Rp7hT63b}aO+NjP z63@Vgj?nX5;u-m<&rfb7JY|RtY58{Jk{M3++n4Hfk{P!BV~iPG79+Lb$=Q5b5#x1b zfv~pxYsTmEBy~gKGKPP=Arf$_jA1QozxeNZ1LLlTYtWH(BcmA7vFstz%*fS2d@&zv zV@P&}-Reu|U>H8RTKlBEgW;@{8%rqYWTfQ=;b4%{>XPF1UJXIlZ9d{ALd?!+O_>mvU^f~r$>-!-O^Zw&^O#E#k zGh{a)s?v$bG%8|Vs=Q5N{zj7I?WSnV0!^9PEjXPyodlh#8)7rLbj5?(efgM(8{#)- zk*Ame;v2bEZ9>dBpZoQXeuy)-d=a7o^AgNYfk~oAZ_hC+PCO_?k6vIJC5t#JtVuIx zx5V`9YE_t@1#~JtJy&Bo93NCk{G!Q}ZaDw`(woanpVsKYzH8;$e!MNTBf53fYo)T=_*5Dat>Pdd+ESDQ&Dj`3@ zPpN!on(bBda8J)Or(2#X?!8)Oep6P&Wzkod<0K&;)23Y}x9aB*a?D>Qe7Y*vM-9j_ zc$im_=L%+>wbh!bPk^$Z+^62V2f$c{w;zV?J%zFMe|#kB(%`J|w6)nE-bj}A`jW2h zD3aBQ?nz953#WjpyI{u`&!wGc?Wwu>7+3U;9-BuoOPO zR?+7VV*M@u0u{;)VR^}lXulYaWQ`8nGA_WPSmI^qWCc_-OORo@5zB~Y@tV)Pq~1tk zsWCbm?Jy~|fa|1gKH*jp0kyx2OmcrgstRc-fBa z_wZ^KHUx5Feyy1`KAmwUAisxojbwGo=E5MW%I%;3ekF$v4BSv$I;4e*G9@ zm7VQ9hALlTNfm2L4Ct-0c3VzhqeyEk2^G@jX2}Mt_M3ra-J@TuPU7}f`tN@%JY`Ox zDW03X{4xJdaR`AeKEJ|K|CGQ6cn$P`f{1MEm+7@e22}R$nvhq*7Z#hx%_)v`hs}1{ zKVk0%;$`oPI+|Feon#N#6dOm|i?MB}j!n;{&$1Kdphimu;_QqJEw#Pu3v59n&)cff zGHh8UPfO{R47+OepY>dXJbUGsWXQ2GdA4{Ywwt!B%(lC$Hqn7pWvlC480$W5z_#Jq zX_R)k#datpH(a(dV*^Ptywb5|>>^oEf#<#jTm9xe#U-j0+bhi;HuLB{+j=b7ul%MH zyGXC~;#1X!?9X1r5Y?QA?2fonD}xJHW+M%s5?DX{*fsOWwxmFRc21>a zr`3`_+vhF?_FXoRot>RFW91OawhH}yx9VdUJAO{n86uXzwiP*{)3=|#poQl}C8|P9ysKsm#T|SJ0*IV{!FxlJb-CK6D*$2nwU+>s9qvb|gn@#MG zKT5ZAiOp;!DYUagX)~Jxt0JqwTG&MvGY#aPUiMXo_-(DcA+~V6A2Ub&8yow%`k{aK z99!$9;z#-N6*m9-k>jt$SJ_9#|Mig7H`sj&P2VcN{$ht}6l+8|Y_qv~BI!PPf7urn z-6UJ#AdVqa==zW*lv5KVbhrI47bl`P?CmZK!MR@1;F0fy=2%KSpUT!3<4}FsbuZVH zIsf4?h5X@{IX63sAAo*panu;8sZBZB91FN_K)9(6$3WZi!~#s015}p|oJqOC*=QRZ zkO|l0JYSqyFr3oogmOn*UhKHZnFo59Stj4&T)aGIlQLw)nY@f3f3-8`e7-2=cdy8t zv-GyiBB}Zw=NqqUne&wNvIgDW^Mg4TckXsG3Eg`AP*uWR81 zH7YB&d~V?ckMP#7{@22RCsx*nOSf~}Ua;KlV>&qnY=_D@mo5&-^hI|~Xb(qu%Bioy zV1yI&ej#-3%~y`?2j1Cc+63n$8po(ao{zKslLJm+EOmVU!wG*fojKKc$f;AW*0Y#C z;>aubNi^gEc}W|+05L}hFN!YX_%f7>7vpa?Vd#V44UfzDTZu&SK5#X%3`pkTy;yCe zXz~%myYS0jQF{`@yS;a~QPP9sg?2`Iyx}788qBL!J`tw#%ImBDkD>Dn#DamMa4ABe zl2GK`+q?I&vW4uK5ke$;3yDM-*&!oDAtRAw3rVDG3ZbkNvcmWMy8q9=d+t5YF}uNn z05aZkpAVbC0Mm`z^YzYfK>RZ{f#)L<;PCF5M8XdwfED|j)S-(5ER!8*sOJPg6ruEA zjyn;Mv5UO#;7kXAG$sEv_|pMCd|OfZH<^IUIX+e=jSWCkJI=JVasc6r^iwBlif0Zzr|j{sv_lL^Og zM}U33{3bh3c!A1`*>i!(f2aUe}Fs5kth6j1PnBC_ch0GNn} zA5isVffxMZKUr2_f!m_QGki5fpc(|Npbek_TkK-+O;apjri@gQ*%liJ5#;{!XqN|O zq`B{Zms0?~xa*#pBBlgPN(pwgwN(e&fGo$jA(w#$FVtjaGp+!$XK${g&szh%oRau5 zt?Yp&QOi{^=8iydx|v9(q%+XuAZo`nC?>@i|$lz(^{Se^S!>t!BUW5WwC^r6)7O}wk@te(SjA;@Rju#mFM?izKakKW3kAVJVr_f`*xxkiR@qXop^MIXW zbG#Pko&p~@jO^C`dIo$0{P1^PtQz;~2@OA(dV)T7SD|&!#{6GCXEBk=bZzfB|(}sZSx?M$P-6O!WM!JgS(_=sb zs<2n&xe4Ih)@2>A)FklzFe3?l{u^*>`RG3uaSAA3vb#Xhp8+bZ%l*Hqeg`gS3g11= zGY`DlulAB>a2{wv-UwnC{{rSj|3bz{3WIJM8{MikiGY{oGBP%x(pL@tlNIbh=|w$W=9vzloVNZ=%NN%` zf5quvojN=~p|51Vy7~BmB;C`#G#~Q=4Qb95YXta#ZjU72d#Mx#dis#E0dNQdRcg2m z`M8CHPWE#13;XVZP&4~DLV97~&B+;S z-oJ40=7xx*rw9Vv(Wy0gmxBgxL`!kb1*5@hP<{9Fk0{_*hvKgr*eQS$Sa^$41x4`p z)5VkNFV(bv3A>tGZA!Saj}53sa^NAc7dAMndpuEWV6 zg23VpHXtWJ1Q@Ww`|$VYSaA5Uq+?&>9)Rs(dcgHlso-h-yAhKQv%r6xcf~%9<$}X* zXjuM`e*$*t37PCPEd={K(=*MMdJZ<|Esq<~r~<#)cf$^J)`Q0*D-Tjojo`hTmBU7( zjo?A(yT)(V`oaId7maj#d;+WI77eZ$O@e!K;O$>I)8M(FlLv+8hF9i!b5-H z5F}Qol+bd49|DUq`f@!$0P<^-e0)Yu7y>Des|lJo2`Od%HUL$fg3vlVJhCM&K;#rZ zT|14u01*~AoFA`Cfqd0&Tx0-gkd%!J_x)D5kh?zBCYhO(nNHkaAA}_-Zf)oUp8Ju!}G`zcf$s^1KQWTVf zG!b%zD9GU@?reBN+#U|Xw)T7>-=FDPO!WmpikyK!_~T%Rsu}pUOvf!q*B0&q|L;f$ z9%%KMARPta97oF_Yoj2p$Fg>{nxi2K&G{L9h6xZKgKeGJ@P`l=(y~n6K?>vtz1dGm zA{}yWS>L#-CmUi||2mo9F$coG;cNb3ya=*aN

yDuvV+q1=ZMb&!vZQ7x3UI!M}j zOsb<`8-&y%PfP9VfWUssz1oiJhHT7}(6K%J5QCR?@wVjyki#(Ep}$+h5Vw~dn@O$n zkW^ED8zGfNNatHehZ*H%h^9+}V!+`)ko>An@LPjF5ascjU6T4bb3s zzvKNP&{{95bV7hAw8_>9o#7}3#hJmi{46A(4Z?NS8-FCAs~g@V_~-Lb(B(JLV=!qb z7+Gvjosx$3_r?dNo4}yel7!H84>)vb-+TqrfrQTX9`}tx6QB$Hj`CB{WN3^^<$8q_ z73$TAG+uF}Lz^TrDO0-)=uUi~vF>R(X!nN*`ju`?XnJmDd8D}x)RH{E5ckChs-zW~ zUs7cP1**H+CU}@aGi<~15D&efC*Zu1GDsgNoP5PwIWq+6#58g)dvY6kST*siQ+Xtm zspx85sdN`wJ63S+P){@z(fTMB-*pcLt+LaM5o*H%p+;fHDcw-)f-pW|xH z^RU>1(x}HW7htdf;T_Z+0BmFU0rTuTAWZBVXy}7J95#N=-!5+)0lVb1G59401-n(e z%FKF>47|&P z#rAbYYIQMT#*b}l2JP4|FrW^+aa0LrdgwvzoTLg27KlgmZK%So&Wy47%++DTmPY=U z)iq%@n*Y|V(hOkHR%VHidzLUov(uYfswR6xhi}5}w@!uE<_EyC zuL5I6g@Rz(IUVWsQ-V? zuU-SGcfz*orKGD5cfq1cd>#}_cEiFQQWgU=Kfo-)1lIqie1rulM5U?fe}?@~?cY3B zItV)ggc?M255tlzyBfatjKSWUl^+>;Isv=#YnH1({|5Wx`s@qBYzFpI;^orrp?O$W zm8Ar-YaVvO9D(jJT7rFXDSTT&{|i$(@a>P^+Je#Z7sa@0d$6v(VC@Lekp1sF?dN;=JgPgBwRrJ=bkBD63)1CEiZKN99-?rqxHmQDY)C- z__{?I7A|T0cl^UOJpA^#IR$7t9&YildY}?Wh8J5F=?z9w;Eq;snH~owT(q_7B<~m# z{;K(~C&QfsU;RF&lAb9KPk4_FqOsNBuNoEolc^eT-s>$Zc?ueEKYP#8-DpjCs6$~U zOivfCr~C6F`uAn{ovoMFdACgAV$q9%G%GW>e%G=F{ir!y>DB{IuBZ(>X&7cw=4B7R zH1J~MzOEyDvcD?)F77&9``Cbl+ATM@Swqdd`d@drr1#Nt0$;u0GMeah(fi);J?%^9 z5|xACvkrvr4^v_A8#7+2Zj#~f>*$m^Nxg8m)~TiBFV`dBi4`}T_i10bgI^&%TU>e^77Sf;ZlWXC}QbU6+f8*NUAyR)|V~i{FoS zjT=vd+sVD2c%hpLhkrdRb3Gvgjww-9@BNqyztT4Cp*flh_ih&%U!BN<5BwpVvu!Pc zKexiPYef{p%U0t>U)YqwSC<+A)orD4{+zKVj#lOH6Yi7US9V(9_wL8qBDvl0W&NlS zsfLemnRsViciBGpj8@G+^6O9VE~Q6m$3A_6uP%t?SmzAEKPPr=Feb*}mKEOIx#|gc zc+W;jfX_T!_m`oQ*QGZsfQ~1Y)WFNaQW&6a+l}SH{ZI6hz|!O7Q_!l zV9rlq2ZBuwIixkyjmRHzUI-oQM}U+&U=p%l5WXa|S9ZcAqQQPw|2=F9p&l&8)1o3f1m8kAQ#gC&Fv zS)y+z|2v61c)abOeMuC#8SGJEYGVtmON8R6Wz?J(<# z{4$cV>d1Q&$=f8x;di=;R72mq`sV9RWXzPp=d(gS$k5~E(v!ph!<`pZR#T4!N3!0c}5QMBWnq)(pvNL%Kdr z53qdPh5R@*Q-AILN93>N*a{!kC*s?GP9c3hT+;bxH;>#@IrEWU*+NdA z78}CKA0XB20Q;4-2S}x(lG%^0^Psx6XDz%x^P;4N$dIK)K2+Ofiz`bS$5Hc;j(Wgh zVbss)sx~-M1jVj55-yP!LBWL;Gjyd+quefh1mD3)qvor+kI0K$KsCg?UB6ZWMzMd^ zd(1;1sJfT&$c_*97bDFu6o!(HsVYUIe26zo{wZQmCv46UBb@LkRm{pRhCx7; zXWD)oXAx1|$Cw{wZ78UP-;Rly^Awb0N*wfgFcrlSYboklr=gTFKPyPuOwX-QY+X{lgwk+j+{?h~qJXAnBzR|yP*@Sa zf~9{(sC4%6y@N<&RCQxk_xY(SsB{0KB2_A_QKqqy=*yGVsN4q^2O0i0s833w`xs$6 zl&QGvhe%5o6yT|0UiyifC_Pj+%zMuh<&la)AB6^^N<~M{*uZb2ZmRg`>}o}zyt9Ev zmZM@&XCelYoUHGoNQOTPPoGOiX$S&tT^-3o9r{pTX||P*8mr!K-0rGGaetM@)SZ8W zvRo zG92P%=lwXOYlZG24!BUs2JKPY}T|sxsQ{{)G5T<|VEw698+5E8&=+#o4h=v4r~kUWJ$j2={LJTu3pxa04U{dHf)2d)*I-sM42|xQ3x0bl z0`2p~V(ag{JLu`Be|(~{qR^tkcjZ2%+(nDDW*jTndx!=~^qd?23ots=-eT}AyX8d>`R*&wl@u&=ze1rBJZ|vBvZbKXIh{$81U1+NuZ(V z4fLJr1zg2l9t;ZNe3N!W9P`S?f3P)N9CNk(kFK$m1V&8K^(!mm0!HGsn^e>z8BFi_ zgQduy0L)h6U2~Uu2L;77^bdmx`$(i#ORc>&OSd$!RXg8@N%XUOh$!HtylsD z^UEj*FK0}}lz|QOG$U0pSjWhn@~f9H4bc`3(k_M=ko1f8VsA@~Q9V*IgJ_>F$GhTYc_5$uaD@(zkNF$d8l*ffl*pl9ur zY8vLwMV9H{=R8c!L1jj5LLp}M)ASLgj#A9ot}*5HtK}H8L)nUSY9&Ta^IdSiZ51Y> z{Z-Y6lvkK5=k1fvabIJe@qp4dtKVYY585c6ZfV7=1PGzlrjz>~a@q5gxtAf|kO+R8nEHZr?gZnV?;_g|+&-*X~&y#gOy!?a_ zu~@D$?-|A<4V3JwA*V49f9B`xy3bEcu6tZhm*G zN03K$U*L#go)4dFiFBL(_7#5FS%K=EWTcQR&bQ!c+glOYGK(tJhoqCT{Hvwob33PH z=R{48d`}UTm78t8Ve?N)HagD-8cet#3p(0dROBWjtF;OugxEr5f1|&SDu<(G-5#I+ z7{88}-B7)F$uN%~YkKHp;}qA{Pc>{)67r6EX578&>wciBi=_9aC1#E)QI*=Hi59h&#_WfvbtAG7tblXWit znr`9fDBGbRJ>U>~Q+CecM_)#hpX|MFVv~Pl17w+ArYw7tK-o(zVukCr!Lm{}@cA81 zw`6(Wy7P4Lhs$al60)wUi;!LN)cX*X^jJ3Ivjl=vm@Avxe8!87e=4gOkn*~|s9g3< z%Vopr2QOsnyeu}pHoug`Z;QKTir32$8DNE{Pu|ET&GE%vt?!k!J#6cc<2WGe^&V}V z=Qt{xnMhx+D;SdvczQ5WTs0xPAa9vOx%E}nqo(NBzws&AHQXlu6U`af$lOcGPgmw; z0r=hJs;4Wm_hqZh+O&Sj%6Y^+&e2_!eFoySU|(35{r8sE9qYL%D^SFdtuPhDN~vT@ zl9Pq8NyjY(jt+`oneZP!FKdcnqt%N)jh=*H4ZbGkeV&G558pU6acB#UHaCfBLL*goysN!)=J;4#H z%fg$mdTwA3h2@F3J#xbu;D7a~m-=Ft9zYSw96xN81Snu?#}Dh)>rD6;7leKAh?DiG zAP9R=W+eJc;VrCh!JSL}{kO1`@tQ~TDUsM6=#lYLyouNw+B!*Hp^4b~(PI(`_G#GO z`ezH$PHEVZvgmBFt#oYcz>v$1%42Lb&1Vh&ssQUedV9LI@;NrH)b7dZ$SZ8+VYNUL zuMRA@FOVa+*nw?I#;(33cVc4;ZB2d_eZYc_$fjLT_=w#$+$wN~e#U0^9SSiw48spETW@EaS?S?Jpp`HKa1IX^_D9L3%1_deD*%!|9%-7VbY z%8wK7z30+!Ul1p#FACel9>-bLj*SNagmKm@J9fN1CvnGaa{8B3&f=h#8+s0K61e9y zp{ufU5;&y?FA4033pnG1ypLT?GB`OKmG}I&0JtBYJeB7=Ksf1rBO4<_Fz%s}F1YFs z1ozzc=+XrzC{E;lYf}Rnfs2Sc z>HZIkyX47=jCoGN8E;w_6zVc@BJraoyz?B~>u*{?PgHeqM$7lAaR)lMm#5N9&Iw+| z{nZ19k&YYVOc&p4=-oHL)i!}oT)1zJtK0}P{YA3CDXGLU&%Cj~Wwaj85wyFCvzU&H zi23D+>qY9(cemYeKga)WCntH}`ePCf{Kh(~nQQ`Jd09ot_cYIzyB+UT2R-%7J^S7Sxf zce)?rj_|=X+h+4|PUw0kl0*USS(4NTxOOqF;MxW2WV0R^K?TM&ayzmcbMbg&qcM*i7SefR9hjN6q2d z9rk(9ZF4ws3$o@(=uaH~)qs0E}nY)voNMf0q@V&tR97F;=R=mhTvMok(><{1uV#-}Q>7c=t19a~{_?;IpnKIlLCShId)}lJecb z1)tCuBU8KKT^=f+T(l zen~1c3F<~y?o0kWODLtJ$7>Y;34`ViE_<;s zLc2+2k~tVb__C>RL$D4(_>q?B?ng%xx{+^M@B^|0OIB;(!U&u2HMVI^Syhwp@?r&4 zD^`neCu=>xSjUjCnsyl1aKn(Wukxkpfr1f12a4e<&Nm_))qA^m>a7VO_N2hm@NpA@ zLd1m2+HZ3L`9;%DtvgnP)us!xKHsef$1Z-|?fhs>X!TID6rXe;q+Tflq)9swVzgJs zw>7R4IzY|}AA)@d0nr1JX-|C!1SDgZtR6vFc3*(i-iaV=toI*Q21OJ4jNj#du8buJ zINyDCWj>D3J$3T6d}Jcwi~{ZPsCWjU=1P4%Q8AlfbJ0*LYA2h}tX-17{VR`9X8+a8 zJExTJy6K&m^Ku0NAhG!DOkNd%(i^FcsjVS!v<{W=^VAUxKfDa7sB9v<*(jF^Y0 zZ29U{E!RfSdWR}9eA7voFK=AF1MVX5o6hun3+W=LWc1!J4e2JF%*q<<@cls0PTy-p z3l9)Roodrf?tdmcDZGEd5b}c%xb-EN=i~w*VLm-pY-fewaZ@?un=2V0@c!x&8ci>6Cv2MNOhE&h3Cu@a-aVLq~u(7CX+OF}jEIMJE-Y3?+?CI&mC+DNdQh!n@i z?L3Rk#Aw|Iv){AZh;_XuC~eXmM1PH5=XCxL#P+`biz6yN5-r&fx7mr0#H(j$`I*em z#Ns0LMy;J;;!@qo!7#@;Vm2%;c+lV%artxVvEzBG#GV-w8D`cRvDVVdrnGU5c-HKN zk!tulk(*oF9Q%)lbXKtj?!CfKicaV<#AS$*%BQ`E-we-^o;(Sw>lu+C<@dP#Ibjr9~3p zC2$UZ*C7SO-If$(8rInNTHW~AH~IR?ko%t?Mn)E0^*uaW>E8Rs36 z>`1S`-jvt8*GOaDVoPWJT}YAzPb#7GbK8I=zZMQe?r`xq0i) zBwepho-1NsNK3zNa8pl@k!}eaj8PRPNr=pwg~@Hxq^S%A^Ww%CQi{*883Qs)LJ3LS z>yBI?eLvX}T7g_4MP^&3U(o+W>US+<3u1ngCi|Bad`s5;*I6>G{?;Z5693zJMgiQK!H~D5wY6tk>Etu1mcyriE0EpX)@who6v^|Xlk4B=mC4;% zuIoJSRmp97d;vrub+V!jo`qJ>AWt~8=#46AkoAw!f+90a$lsr88(-=*B_~?t4C44K z$Tog&mUU+=$g{<#zTp}i$=VN68Ca`-1Ef zvO(|-Ul_fRd^iNxC074}T)}U9JMGD9@>{B#CO*57oPGqMy^_;R4y=ejj&N-!>vmNo z6zg}AL*gW^kskJvIiCGXs@9*$6FipEuY3l{iQc@0>XTo|tju3yPq(JXR$4t@+(19b z%roVgxs?m#Ipizi1BJijrVHMc=bg65Q~H4qV7^=AvC>lJtmi)2(JjtMSKxpwX?QBm zNai$U@Ap3EDfTo)53g|Nh?*$n?&`A>e8XobSq%#UR}{o4xiMImYnlK`QHjugls1*} z>5=d|m%}tlX}`GW|Iq`ArvAv+++8;1cf%Klhr1lgW3>Z>bpe-Bm@)Mxkf}(qog!Zh z`lv`rtn&;lbWx>5&DoZND%2>T5|f(hSv5*w)vt>ZZJLy46W4Lx(94v7WwOoI#UKNkI5a2bEgD?gdB~9yeMH=S0Y&uZwi0krPS3ZZ;JGE zY;@yWAIf%zK-Xf9AEm?R4hs4wkg_859QCs=lyXivB4Yy*LovKGRaur1LlFxuC`W&+x_E6SQ$RxQf1tNhD8!bDVCSY(ine}$eiS^9vaP%4+}!?z5+%S} zv#wA?5u7Dr*}_jLC7-Y(aAXOkG9X4R6#txJF3=265GkX8zv>UEabHkgqxUnW*j1F0 zA9sd$Pt;S$I0LITvl@p55JGvQeBiSOPN^3#V$&a{9pdc z-#wI$s$grDE$qv<_`1wFsIYaH@+%>V>sS{i7;U zQ~#tNC0tUa##0OXsS`TXtI-u<^WH|(-Tv$bx}Pz%Zq%p6Th)~6@||1j3^1p*{yH{v zw%Czcs|LQ33cpU3bsvleWw}rdbf;{7rg~8KbMGG9iu9z)X&*u8w0lwwm25Z#!Cq9? zOUT1TA-+`49g@N8WR%S!}i^O8$|_V#?*zG zL{o>Xc zFSX=*fVzZuA9X0t8m8{nN99bN)e@WSr#gK*11?<~qGBqaEs$vA)Rna3Cx!2QrRLNE zp+`AWRI4k4@t)3qsAb0@>?iuys6G?P441(*>OkKAwC00<)N_El{13+XXd98)KReNa zG?9_7FEWje(>$U8ljY14G(}K|T(Gqatw=OC7OV=T8D2?hJv@P={WDVRU=5;aKmRW( z$PSjJtwx_hLyoSK3vq0keGAI2sg+He7t%1(B&gD^`37_bwy4uit*yMIglp3n zdCH(oXB}FXV)WC7Q9au7)ilEZUK85cfaeLB*G*~u%Qg@G2Aa{-=D=fblFevQ8g|=5 zu9md7+I#SDSu0uq`k7dXp%rbSBBsMv+nUBc@&3Dn{#Dw8A4a85KiJdy$%4VbSDk3N zLIpf?cxT#)17i(=L1&uz`C}#Er>-e-GR0T0@05;J7?mIuwNTX{)V z)00-1-LzBh<3)S59C&HJ&5PEbw}-hd=1tQ|77Y5a??XFte_EQw^rfBeJy~`b>rXqJ zF8?&25JV$V-fE;+2GPDiZ*QL|2&HugmEjg%h0%}%OLBHy7_GQcP-S;4oYn?Cb?&WM z6b%6UG5>b;E=_nj`{nWASlS6*@PO$1`?R`kG^X<5LmKaIqhTZ{iFVV-cwe+VgJxV1 zIIK~TO*`EdwC&JdKzn;UnDg4dh$cpVh?5>DqqU75QVjH}q=8LETMFH)X$`-_t|}+I zq`f6|Q_pL*(sUf>?{XK~Y1gmX$lIj7qfs;5ZTyvbX>VfE2h72rXeGye7Y9~9(WIeQ z^>fFEX|EHb{?3PgrLB(mqxef#Y4akG#=m)gX(20#7|`lJT6vlDQMup$Xlv4@0?#Y*^T`eQ%0BK|jG^qbm_T%MA1^dECW2Tv=`(+v%enz~v`)1Ufe zpSUG1LpRrMcx`F|q~mVi`_dN=rGE)=%6@PTL3eWPaV=fP(07VOXCU{)N?<|NQiaB1QtG*mcNh#Bzcl_F3 z|9(}M-sLQuRshwb2Vv|)rspi^4@^iYcaB@q$#)Ybs_3@#LEZOV(;JTT1iqpwn3)s( z_RrO{8`&;&p1bFJ#Y;WuGIMBgvEd9~Xa(Jwa#_v8t&-jzu4|HsuBN}K z!`g`&SJU4H^&{%)YUr&^&UgAg)X+uyT2@Z$)zVMjS79AlYNC&Q%B*u=@1#>(UI%W5 zex`pV#j&Rz4bl~O36lrtQM!QH@6f2cQM%+qNZ%*vZ*+t8m7jjR^K@2IzEJt8AM~8F z4|)Sje$YAAS4mv_PrCE;urPXIm3}mdw(h0BLH9n9b*&1#N$;hYy*q;2rF-z_tei&O}#^<)hOQuz)8FAUQ$z?gBjOy^+TWy|a7|#Ndm7W`(XT-m> z3MlC~&uFXgnS3ZI&A2uEq4~tF48zdk?B?rh07e_u^!cAhKn6N${z;w?m{Ek@e7uo? zVt7WrQ*pSDW*||}1fIuahRB~I_`MSp2IPU|T?~oBupS?>MnX9Z1y4oN;Snyw^ir_S zS!YGY?P#>jT!|thH1O8`)p{icd>EeF$*aOBxSIWDbxD))+p-*IEoZ>UnK8L+s$j&> z|K#X<@Z5x9yxkV@4rs#=)GTW4nz3Qzq^-Wp{&I~`XlcG)k?q9LyGA>2)a1?p`L9UC z3;8iRkOa9BTqq;ud+Ur+SR})9^R|rWSQI0sgL%9?B8Cw;(3x(}j%5@bMjnPC&gUHjS9#}LlBY@dE>fbl*W@r-+-ixFlA188xN6%nA+dG7i$kmGS9rb9NMxZ%?x~wGL0{^WtJP`B%&flp9hc=ICbZ&sz5I#i zOxUoH->=LvX7iqV4s7@Z^Y8i}6UWjD<}%{tQdU$o^D8^wjs>V=vg4vt40hizlSca~ z_OF_l2R3{cs-l{iS`5PTEa7*|%OX3d4s0iL(7CXB=xrx6;3VLKU-Jj1-B|J z1GCP=e1H5C4`C@i?0$2O2w{Cc?z9fQ^WBUpm|0lUsJXqIGA8#9C>%i5mH zaAk{QSse!}*|l;sR!wgEyQc~amcO$9kh7*D%YEP34h3tEs^wD$%OnH-hZ1N$?$9MXjT0=TZd{N~#NIHWR|M`UY zLUk_7A2e4GB%RNS6R(#?gcY!MQFv zr1eP*X^lmwz2DYVy~esb^QR?RWs7x)e)&sd^fv2O^&bCJ`!?$oleg*O%nqyRYXL(j zbeHAu&^GxlnwOoaoTRLKT!1Z@d1)cnS%9s{>tCsVMUee5=5(6Sq#%2Ug#Mo{f0C_# z>7JOk#%cCPHvS_ILX186_iXvlrX+jo$h_afKq+>K8sV!bzcf2%@j5E}#Rc|9d%+5= zKOnZhl!4ftUIaU}U%7GG9LuhvJ+;^JB(e+a$cx+7RCaxIc2j0IgFQ5PK<(kM*k|*3 zhE^QdY_rcMgPvOQ>_Mi*dChlfY`5EHJD-ngvhg*_LTYi^>{uzo9QRZ`cF^60e`!Mc zY?ZW;-Pn8j?Djj=j|Z?<*it!!mJ=0LY{k(KuLsX<*{$GUm89UCY;IFU2^Z?lrhRzG z5^8m4UuU1A_3-+zC5YKt9g_iUeFdjHhp|95|KuT&>mEVuy_G9(eY|e5Cy$7!)Q{a_ z`&ws&-i{AtoAc@f!;Qn)=fthlt*xTiBF_y{?qOrtz4_0i^gLqNv^%stcgp{zfX~UZ zQX+}$Nf9lv$(m&Lbn*T|y;BZ5LSSFCA)$aR+^SGVx%QOZmx=U_yHv_ z6Vv}T-mL#$PX!rIY!5r}bKj}rs88%~Z&c+DZGB;P%PP3%iHx(w?XLVaH~hx#Vm)2s z*#2OjdQevpk@|z3eV`;u0j#j43@oy%87u6BMAyH+Wj5Fs_f=#RQ#RQV>kJdo!&~g7 z`-cZQlD66R!yYGX^X{|xY$4haoqU`a(pL27{bQUO>!y?+1wx!Rw;DEu^G|Uuc^rr+ zXozvLLwnrHA!j*$mzI2%3nVxd6~9h0zDRORB*(tj4oY!&!k=$Sh+g2_`K7aH6CuO- zc(cZ;UJAq!I_%(tt%h-qYbX`pz6R&?eey9jlt*$RKP!xa$B`U|grn!O$5EVGYp@(b z5YK71Duho&<2k|?24nq61Wrg;G2HYlopWPCGpmKj;=BmStx4Tg;{2={d!PREB4_x% zgOO^j7Dpsi8=rVloAdVlq1%VNv^n&bw%-3z^*C4bO7MDa#+=g3)eo4*S2)KvW3z7+ zUE$DNtjpZWu5r2pKNm~^T{yR=zbNK9-QZX}GmsbM_;U)j6v{(t{W%}QJzp!W_;Yp( z?I<1s0i2zg-lg}0Asm2Z{GAc!Tb#_eH_9gHSWbM{=l+qOX&ejQOt}olT+SU#FdH9R z$N^_ccDK=rI7TUKt-)W#99wWb@RxcO=ct{pW~M_e=WxTYLYYe~XG4YLarHq1N65hF z%gymd4)`#uLsq$!lYpykc~{)Z`PZ>~Y&yS<^PMk38l=-c-iyZ3U ztSi<$EEjmkAqmL8$w75-mUc!^u3J5v|E;>7D3v~r4d1qks@ot(_X07!l`zBU+~ZAr zZZpGqugHF^*fP(dSY-8^ffhI&r!5<2%T_rMml2zk-3^Xc6XsiI_BQ9A;i;_?uwBkR zvbC&rZI82kX~=V{`7rme+F-C%2p{)Vv|X1XREX;fgs$%q#JF=(_8ELQaqiQx^A(u=u8p26Jx2h#)%_ed_L`uTM&xx3uY$=F7#-dJwud$}jC??2$aS>gFKx0lSV zaX17MTTJKPTOE>M$2{VyY6DA;4(D)X%AP?hzvXei+TGnEmp$Pc+ui-UX;Q*nULhE# zFrRVf%3gHVAFkk<1f+91jVrjI7O5w>$d_E!`t`o{)HmF;H5wp|w}t!DjvY&EY~g-& z)}j~fw{e?Xf8<9xbaK5q-Tcc^#<-6@tJM2|C%8{lKOUW9&2UXVMsLl$nc=<j#NzZF9TqAM4%iIwGelIq$EPa!hXA zJoQx%L|E>P#UakjpopAn?3JOVv!Zfq-BAg%lHziQfa{hMG2(JSpDvgBxC?SA&A|1Q zPiVPObp79{BY3%%HDT_Hm`297qR!)GZsP#ienOt>PaaKW$TInY{;V05a{Gsi)0?F`@f^ZZV7t~EYq zR$sfw{Z)%L?kBs-m2rGe{0}~sYX*EU4|?PwcPlUVN4<-eoN|FoyZCP(xdc?QSHYn` zxocac8-61(a*w_4>3%vMEBDI2U~KUIeK~Kl)H`AViE_U*?;t`RWy#g1O8rRQek`ZU z`9u7yR4AunV|u`A_f+nKb#o$_S|K-TlGJh4p;B&ZKshmw(kORZ%m2uGN{^gYP}k7= zwjQ~6+v@ROKMcrekhZ`x#h>N0z6*{e2Mo&@eVfJhlzo*OustVh`Qn=#RK}KGu{0(3 z^2B(J*0X6jWeo$@Ld=q!O1P0&DDQ?`XP`iC=e=#Y#DMXbt*sro5bVgmMb$&{do%V5 zFRKH74uI z>(4)V#qV}mo?w4!LM+2bo^AUn?9F>qdEUjUvr14id557-WphK;@=tZjkDpX@mjADT zGyjJI|Km7`9HY>&$!x;7wy$$rqUiH^f5ylaDm^NiMv=`~NLiJeLG4sSLv)H9xkI`( zGRc*&xg&Ca-Ev0OfAIS8`FK5^ub0uzFZ2RK57ebS&Ek!EphFx|SaQo3DQ@<-dBWKb zQK{^BTS_o;4WN5TbJ<8OYp+mtHX9*(1%17HQAo3>_+d;!3`(Av=536*h3sOSrYDMc zh@L*Pd&l@)G}0(vK{6kr7#qH5O4}ne5jwT2_hLG#JZ(6{B$?33UskRtc=S~mj|JLrH_^rL(XdDBUMoo8@(xX5uVdRVo#UfUm#5+eM{S3+ zH&iU=sa>i=953nhie2A5}SLbqFiwisibk9L6Qb zH{T42v&TwjY%J48PUB^YS&nwBGrrvjW@Yc3ahSBe+~)unJgL0cRyXB>Jq6OZ@iY&t zyox}&98cWr-&AmN`5gY>=DRFW<&EF@ngts-`QV8_0db|%=tNC@w{Sm&a7R6t}DY&rjzH+3}Q*2e9xH9oqA>P9qRiLvkFb|-f~Y^yk=a- zFEk(mZTPtCTPcgA4qVG_tvj37gJUJiiur55;cvM=&y>{<;;Xf?iAvl_`~xJXecGn5 zT~76mYp13$-cQ{vJur{kov(!z_y53pw$YY@2}@WtQimu?3Bg*y3z0-X1Y|Giuw(a% zK$q?hQR5m(@RSYf$5a`RF!WpDRR0Qk;@M*2{@Y>Ld$sly+5rcL2LAU>P7x@yQk@qm zO3-6p95ThGf!}>gwF~MOtb`F^VTiTMXVuAv6~`lY6ndjpTh$a?4e!w?0X=v;p|5=k{21c#aT*@L@g6wt(CGo{Fj~f}^9MmOA@j${m!bZY zk;Ut}Akd4K-BS<{0puJ*)8h~uHeI|Cnx_yA2~|eA7Uj`!XrQm>I_)}83@1dRl>P=$ zJs*wSiaW4V%lrL?GfBW|>YO#JN(HO)u(-1~4cIDwh&6sohp%7PbhNZQg{HG=cWT?S z;Gf?F%Ubs20)^pTF$M(?UQq5pTk{MyJ4pp{EQ&x)`Q?I$a0&dsp4ru?UkaRxWQm&e za?l>vt}GbhL%sij`^AnGaQ@8svyN|SS94k>e$xI3))~kArP4maoTzSuHtQ2ykqi3A z&$}MRP38=C6g0rZzOpH9OC!jrOs-!u^%;y;hIXB3Zw0PJYVW4+-C*OgP?M(91Gd!u zW{2!SXvmW^EO63Ll#m@sxVs}J|+YdN6112YGhKoeVt-{1-@@pyNmA~-p8I_GhXT95!jNwL7 zdUL;vapXpF$};w9P8OBqFLxfKR#C~Kc=CljCk0YY5la(HRv;G#gL2-E8l-Q}Y3Ah`4Py83l95V)7IC<`e?xqp7O_|z;lJFjM<9qlaNw#QkxVH~ zuWHmM(2%dV{mx#(F|BleYKh1kOSGco9wGBp4kt6T%*eUG6Ws=`7No7~3yqd)MaEJd z8oX_@B97nkY+{R#5Eo{6{E6-(WI^j>z19YMvLY)t>(I#}Ep>$#`))WAnR*vn=S&a6 zRkt^At@R+2t+#qRpL>$`v-(R~bboTMP{ue~HIguQg+?X4h$OKx(z!uJY+_mYA}@k_ zi*QA=YLlho2%~iU7fQ%ILd(Z|nV<}E<-xiS!9n>XebC88BmV<2w(T}S>WyS1J;pe3 z>R&SQrvJ8eaTjU&bL3+Y;~N>e#N4BhFhVY5n>3i$jF2_V-+LEAMoF&gDA%uHoG=fk zbDi7gi2nKTEGg*)vhpWw={akKbV{nmUR4#Kvyz((dR#^5`vyIzNv`YY9O)0mGQ(o@ F{{U`{l^6g3 literal 0 HcmV?d00001