From 1f5ded90e511dd01134b58d96c7139af517de627 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Wed, 26 Jun 2024 16:55:56 +0200 Subject: [PATCH] fix apache arrow dates (alternative to #2096) --- package.json | 2 +- src/options.js | 8 +++++- test/plots/arrow-dates.ts | 9 +++++++ test/plots/index.ts | 1 + yarn.lock | 52 +++++++++++++++++++-------------------- 5 files changed, 44 insertions(+), 28 deletions(-) create mode 100644 test/plots/arrow-dates.ts diff --git a/package.json b/package.json index 918dd27058..ae95183e6b 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "@types/node": "^20.5.0", "@typescript-eslint/eslint-plugin": "^7.2.0", "@typescript-eslint/parser": "^7.2.0", - "apache-arrow": "^15.0.2", + "apache-arrow": "^16.0.2", "c8": "^9.1.0", "canvas": "^2.0.0", "d3-geo-projection": "^4.0.0", diff --git a/src/options.js b/src/options.js index 0a909c2c9e..9e543e42ca 100644 --- a/src/options.js +++ b/src/options.js @@ -584,7 +584,13 @@ function isArrowTable(data) { // Extract columnar data function columnar(data, name, type) { - return isArrowTable(data) ? maybeTypedArrayify(data.getChild(name), type) : maybeTypedMap(data, field(name), type); + if (isArrowTable(data)) { + const column = maybeTypedArrayify(data.getChild(name), type); + if (Array.isArray(column) && String(data.schema?.fields?.find((d) => d.name === name)).endsWith("")) + column.find((d, i) => d != null && (column[i] = new Date(d))); + return column; + } + return maybeTypedMap(data, field(name), type); } // Arrayify arrow tables. We try to avoid materializing the values, but the diff --git a/test/plots/arrow-dates.ts b/test/plots/arrow-dates.ts new file mode 100644 index 0000000000..52bb49b221 --- /dev/null +++ b/test/plots/arrow-dates.ts @@ -0,0 +1,9 @@ +import * as Plot from "@observablehq/plot"; +import * as Arrow from "apache-arrow"; +import * as d3 from "d3"; + +export async function arrowDates() { + const athletes = await d3.csv("data/athletes.csv", d3.autoType); + const table = Arrow.tableFromJSON(athletes); + return Plot.rectY(table, Plot.binX(undefined, {x: "date_of_birth"})).plot(); +} diff --git a/test/plots/index.ts b/test/plots/index.ts index e224ca5c1b..6920b1ba0b 100644 --- a/test/plots/index.ts +++ b/test/plots/index.ts @@ -12,6 +12,7 @@ export * from "./anscombe-quartet.js"; export * from "./arc.js"; export * from "./armadillo.js"; export * from "./arrow.js"; +export * from "./arrow-dates.js"; export * from "./aspectRatio.js"; export * from "./athletes-bins-colors.js"; export * from "./athletes-birthdays.js"; diff --git a/yarn.lock b/yarn.lock index 92b4defaa4..55cd82b36b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -625,10 +625,10 @@ dependencies: shiki "1.6.2" -"@swc/helpers@^0.5.2": - version "0.5.7" - resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.7.tgz#36c05f61b412abcff3616ecc8634623bcc7c9618" - integrity sha512-BVvNZhx362+l2tSwSuyEUV4h7+jk9raNdoTSdLfwTshXJSaGmYKluGRJznziCI3KX02Z19DdsQrdfrpXAU3Hfg== +"@swc/helpers@^0.5.10": + version "0.5.11" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.11.tgz#5bab8c660a6e23c13b2d23fcd1ee44a2db1b0cb7" + integrity sha512-YNlnKRWF2sVojTpIyzwou9XoTNbzbzONwRhOoniEioF1AtaitTvVZblaQRrAzChWQ1bLYyYSWzM18y4WwgzJ+A== dependencies: tslib "^2.4.0" @@ -642,12 +642,12 @@ mkdirp "^3.0.1" path-browserify "^1.0.1" -"@types/command-line-args@^5.2.1": +"@types/command-line-args@^5.2.3": version "5.2.3" resolved "https://registry.yarnpkg.com/@types/command-line-args/-/command-line-args-5.2.3.tgz#553ce2fd5acf160b448d307649b38ffc60d39639" integrity sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw== -"@types/command-line-usage@^5.0.2": +"@types/command-line-usage@^5.0.4": version "5.0.4" resolved "https://registry.yarnpkg.com/@types/command-line-usage/-/command-line-usage-5.0.4.tgz#374e4c62d78fbc5a670a0f36da10235af879a0d5" integrity sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg== @@ -900,6 +900,13 @@ resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.6.tgz#818551d39113081048bdddbef96701b4e8bb9d1b" integrity sha512-dJvrYWxP/UcXm36Qn36fxhUKu8A/xMRXVT2cliFF1Z7UA9liG5Psj3ezNSZw+5puH2czDXRLcXQxf8JbJt0ejg== +"@types/node@^20.12.7": + version "20.14.9" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.9.tgz#12e8e765ab27f8c421a1820c99f5f313a933b420" + integrity sha512-06OCtnTXtWOZBJlRApleWndH4JsRVs1pDCc8dLSQp+7PpUpX3ePdHyeNSFTeSe7FtKyQkrlPvHwJOW3SLd8Oyg== + dependencies: + undici-types "~5.26.4" + "@types/node@^20.5.0": version "20.14.0" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.0.tgz#49ceec7b34f8621470cff44677fa9d461a477f17" @@ -907,13 +914,6 @@ dependencies: undici-types "~5.26.4" -"@types/node@^20.6.0": - version "20.11.30" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.30.tgz#9c33467fc23167a347e73834f788f4b9f399d66f" - integrity sha512-dHM6ZxwlmuZaRmUPfv1p+KrdD1Dci04FbdEm/9wEMouFqxYoFl5aMkt0VMAUtYRQDyYvD41WJLukhq/ha3YuTw== - dependencies: - undici-types "~5.26.4" - "@types/resolve@1.20.2": version "1.20.2" resolved "https://registry.yarnpkg.com/@types/resolve/-/resolve-1.20.2.tgz#97d26e00cd4a0423b4af620abecf3e6f442b7975" @@ -1250,18 +1250,18 @@ anymatch@~3.1.2: normalize-path "^3.0.0" picomatch "^2.0.4" -apache-arrow@^15.0.2: - version "15.0.2" - resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-15.0.2.tgz#d87c6447d64d6fab34aa70119362680b6617ce63" - integrity sha512-RvwlFxLRpO405PLGffx4N2PYLiF7FD86Q1hHl6J2XCWiq+tTCzpb9ngFw0apFDcXZBMpCzMuwAvA7hjyL1/73A== +apache-arrow@^16.0.2: + version "16.1.0" + resolved "https://registry.yarnpkg.com/apache-arrow/-/apache-arrow-16.1.0.tgz#7aa8d0d436dd0995d9dc5c36febf380d5b207209" + integrity sha512-G6GiM6tzPDdGnKUnVkvVr1Nt5+hUaCMBISiasMSiJwI5L5GKDv5Du7Avc2kxlFfB/LEK2LTqh2GKSxutMdf8vQ== dependencies: - "@swc/helpers" "^0.5.2" - "@types/command-line-args" "^5.2.1" - "@types/command-line-usage" "^5.0.2" - "@types/node" "^20.6.0" + "@swc/helpers" "^0.5.10" + "@types/command-line-args" "^5.2.3" + "@types/command-line-usage" "^5.0.4" + "@types/node" "^20.12.7" command-line-args "^5.2.1" command-line-usage "^7.0.1" - flatbuffers "^23.5.26" + flatbuffers "^24.3.25" json-bignum "^0.0.3" tslib "^2.6.2" @@ -2150,10 +2150,10 @@ flat@^5.0.2: resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" integrity sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ== -flatbuffers@^23.5.26: - version "23.5.26" - resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-23.5.26.tgz#01358e272a61239f0faf3bfbe4e014f3ace9d746" - integrity sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ== +flatbuffers@^24.3.25: + version "24.3.25" + resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-24.3.25.tgz#e2f92259ba8aa53acd0af7844afb7c7eb95e7089" + integrity sha512-3HDgPbgiwWMI9zVB7VYBHaMrbOO7Gm0v+yD2FV/sCKj+9NDeVL7BOBYUuhWAQGKWOzBo8S9WdMvV0eixO233XQ== flatted@^3.2.9: version "3.3.1"