From 8f00cdb852d8ece974ec8feacbea32d1f938faf1 Mon Sep 17 00:00:00 2001 From: Eva Roddeck Date: Thu, 15 Aug 2024 14:18:25 +0200 Subject: [PATCH 01/44] null check for name array #10343 --- .../java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java | 2 +- .../java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java index f68957ad060..80e32184731 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtil.java @@ -123,7 +123,7 @@ public static JsonObject getPersonOrOrganization(String name, boolean organizati if (!name.replaceFirst(",", "").contains(",")) { // contributorName=, String[] fullName = name.split(", "); - givenName = fullName[1]; + givenName = fullName.length > 1 ? fullName[1] : null; familyName = fullName[0]; } } diff --git a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java index c3d9fd8fcd3..1f1a2d7af00 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java @@ -89,6 +89,7 @@ public void testName() { verifyIsPerson("kcjim11, kcjim11", "kcjim11", "kcjim11"); verifyIsPerson("Bartholomew 3, James", "James", "Bartholomew 3"); + verifyIsPerson("Smith, ", null, "Smith"); } private void verifyIsOrganization(String fullName) { From b335c36fdabf01dd78897e7c5c60cde4151740cf Mon Sep 17 00:00:00 2001 From: Eva Roddeck Date: Thu, 15 Aug 2024 14:29:04 +0200 Subject: [PATCH 02/44] release notes #10343 --- doc/release-notes/10343-trailing-comma.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 doc/release-notes/10343-trailing-comma.md diff --git a/doc/release-notes/10343-trailing-comma.md b/doc/release-notes/10343-trailing-comma.md new file mode 100644 index 00000000000..03bd18715d7 --- /dev/null +++ b/doc/release-notes/10343-trailing-comma.md @@ -0,0 +1,5 @@ +### Trailing commas in author name now permitted + +When an author name ends on a comma (e.g. "Smith,") a dataset cannot be properly loaded when using json-ld. A null check fixes this. + +For more information, see #10343. \ No newline at end of file From 91377ba38820f722c89aeaae820dc63e6ae76b63 Mon Sep 17 00:00:00 2001 From: Eva Roddeck Date: Fri, 16 Aug 2024 09:47:03 +0200 Subject: [PATCH 03/44] added another test case + normalize String for assertion #10343 --- .../edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java index 1f1a2d7af00..d772ba2b9da 100644 --- a/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/util/PersonOrOrgUtilTest.java @@ -90,6 +90,7 @@ public void testName() { verifyIsPerson("Bartholomew 3, James", "James", "Bartholomew 3"); verifyIsPerson("Smith, ", null, "Smith"); + verifyIsPerson("Smith,", null, "Smith"); } private void verifyIsOrganization(String fullName) { @@ -107,7 +108,7 @@ private void verifyIsPerson(String fullName, String givenName, String familyName private void verifyIsPerson(String fullName, String givenName, String familyName, boolean isPerson) { JsonObject obj = PersonOrOrgUtil.getPersonOrOrganization(fullName, false, isPerson); System.out.println(JsonUtil.prettyPrint(obj)); - assertEquals(obj.getString("fullName"),fullName); + assertEquals(obj.getString("fullName"), StringUtil.normalize(fullName)); assertTrue(obj.getBoolean("isPerson")); assertEquals(obj.containsKey("givenName"), givenName != null); if(obj.containsKey("givenName") && givenName != null) { From 8db46f5e832eec2a161a30a1ee6639c687843e67 Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 18 Sep 2024 11:52:43 +0100 Subject: [PATCH 04/44] Added: optional query param returnExpiration to token/recreate endpoint --- .../edu/harvard/iq/dataverse/api/Users.java | 16 +++++------- .../edu/harvard/iq/dataverse/api/UsersIT.java | 26 +++++++++++++++---- .../edu/harvard/iq/dataverse/api/UtilIT.java | 11 +++++--- 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java index 1f5430340c2..c1a7c95dbff 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java @@ -24,13 +24,7 @@ import jakarta.ejb.Stateless; import jakarta.json.JsonArray; import jakarta.json.JsonObjectBuilder; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.DELETE; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; +import jakarta.ws.rs.*; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; @@ -157,7 +151,7 @@ public Response getTokenExpirationDate() { @Path("token/recreate") @AuthRequired @POST - public Response recreateToken(@Context ContainerRequestContext crc) { + public Response recreateToken(@Context ContainerRequestContext crc, @QueryParam("returnExpiration") boolean returnExpiration) { User u = getRequestUser(crc); AuthenticatedUser au; @@ -174,8 +168,12 @@ public Response recreateToken(@Context ContainerRequestContext crc) { ApiToken newToken = authSvc.generateApiTokenForUser(au); authSvc.save(newToken); - return ok("New token for " + au.getUserIdentifier() + " is " + newToken.getTokenString()); + String message = "New token for " + au.getUserIdentifier() + " is " + newToken.getTokenString(); + if (returnExpiration) { + message += " and expires on " + newToken.getExpireTime(); + } + return ok(message); } @GET diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java index 0189ffd6e58..1003c1a990c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UsersIT.java @@ -23,6 +23,8 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.Matchers.contains; import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.hamcrest.CoreMatchers; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -371,23 +373,33 @@ public void testAPITokenEndpoints() { .body("data.message", containsString(userApiToken)) .body("data.message", containsString("expires on")); + // Recreate given a bad API token Response recreateToken = UtilIT.recreateToken("BAD-Token-blah-89234"); recreateToken.prettyPrint(); recreateToken.then().assertThat() .statusCode(UNAUTHORIZED.getStatusCode()); + // Recreate given a valid API token recreateToken = UtilIT.recreateToken(userApiToken); recreateToken.prettyPrint(); recreateToken.then().assertThat() .statusCode(OK.getStatusCode()) - .body("data.message", containsString("New token for")); + .body("data.message", containsString("New token for")) + .body("data.message", CoreMatchers.not(containsString("and expires on"))); + // Recreate given a valid API token and returning expiration createUser = UtilIT.createRandomUser(); - createUser.prettyPrint(); - assertEquals(200, createUser.getStatusCode()); + assertEquals(OK.getStatusCode(), createUser.getStatusCode()); + + userApiToken = UtilIT.getApiTokenFromResponse(createUser); + + recreateToken = UtilIT.recreateToken(userApiToken, true); + recreateToken.prettyPrint(); + recreateToken.then().assertThat() + .statusCode(OK.getStatusCode()) + .body("data.message", containsString("New token for")) + .body("data.message", containsString("and expires on")); - String userApiTokenForDelete = UtilIT.getApiTokenFromResponse(createUser); - /* Add tests for Private URL */ @@ -418,6 +430,10 @@ public void testAPITokenEndpoints() { getExpiration.then().assertThat() .statusCode(NOT_FOUND.getStatusCode()); + createUser = UtilIT.createRandomUser(); + assertEquals(OK.getStatusCode(), createUser.getStatusCode()); + + String userApiTokenForDelete = UtilIT.getApiTokenFromResponse(createUser); Response deleteToken = UtilIT.deleteToken(userApiTokenForDelete); deleteToken.prettyPrint(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 4e20e8e4c33..8aa89bd0dd2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2813,10 +2813,15 @@ static Response getTokenExpiration( String apiToken) { return response; } - static Response recreateToken( String apiToken) { + static Response recreateToken(String apiToken) { + return recreateToken(apiToken, false); + } + + static Response recreateToken(String apiToken, boolean returnExpiration) { Response response = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .post("api/users/token/recreate"); + .header(API_TOKEN_HTTP_HEADER, apiToken) + .queryParam("returnExpiration", returnExpiration) + .post("api/users/token/recreate"); return response; } From bdf626b15381fd61921440fd6e5a673e20447184 Mon Sep 17 00:00:00 2001 From: GPortas Date: Wed, 18 Sep 2024 14:18:21 +0100 Subject: [PATCH 05/44] Added: release notes and docs for #10857 --- .../10857-add-expiration-date-to-recreate-token-api.md | 1 + doc/sphinx-guides/source/api/native-api.rst | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md diff --git a/doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md b/doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md new file mode 100644 index 00000000000..b450867c630 --- /dev/null +++ b/doc/release-notes/10857-add-expiration-date-to-recreate-token-api.md @@ -0,0 +1 @@ +An optional query parameter called 'returnExpiration' has been added to the 'users/token/recreate' endpoint, which, if set to true, returns the expiration time in the response message. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 117aceb141d..370cd889cfc 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -4412,6 +4412,12 @@ In order to obtain a new token use:: curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/users/token/recreate" +This endpoint by default will return a response message indicating the user identifier and the new token. + +To also include the expiration time in the response message, the query parameter ``returnExpiration`` must be set to true:: + + curl -H "X-Dataverse-key:$API_TOKEN" -X POST "$SERVER_URL/api/users/token/recreate?returnExpiration=true" + Delete a Token ~~~~~~~~~~~~~~ From 95859251bfab94c0071119fa7e4bbcd4fbff7cc4 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 18 Sep 2024 16:57:58 -0400 Subject: [PATCH 06/44] create 6.4 release notes and add about half the updates #10853 --- .../10169-JSON-schema-validation.md | 3 - ...se-support-address-in-system-email-text.md | 4 - doc/release-notes/10341-croissant.md | 9 - ...3-add-thumbnail-for-featured-dataverses.md | 5 - doc/release-notes/10508-base-image-fixes.md | 12 -- doc/release-notes/10517-datasetType.md | 10 -- ...k-functionality-same-permission-as-link.md | 2 - .../10606-dataverse-in-windows-wsl.md | 1 - .../10633-add-dataverse-api-extension.md | 1 - doc/release-notes/6.4-release-notes.md | 169 ++++++++++++++++++ doc/release-notes/7068-reserve-file-pids.md | 9 - doc/release-notes/8129-harvesting.md | 18 -- .../8578-support-for-iso-639-3-languages.md | 17 -- .../8581-add-crossref-pid-provider.md | 3 - .../8796-fix-license-display-indexing.md | 1 - ...pefiles-under-hidden-directories-in-zip.md | 5 - ...1-CC0-waiver-turned-into-custom-license.md | 6 - doc/release-notes/9317-delete-saved-search.md | 4 - doc/release-notes/api-blocking-filter-json.md | 3 - doc/release-notes/make-data-count-.md | 3 - doc/release-notes/permalink-base-urls.md | 10 -- 21 files changed, 169 insertions(+), 126 deletions(-) delete mode 100644 doc/release-notes/10169-JSON-schema-validation.md delete mode 100644 doc/release-notes/10287-use-support-address-in-system-email-text.md delete mode 100644 doc/release-notes/10341-croissant.md delete mode 100644 doc/release-notes/10433-add-thumbnail-for-featured-dataverses.md delete mode 100644 doc/release-notes/10508-base-image-fixes.md delete mode 100644 doc/release-notes/10517-datasetType.md delete mode 100644 doc/release-notes/10583-dataset-unlink-functionality-same-permission-as-link.md delete mode 100644 doc/release-notes/10606-dataverse-in-windows-wsl.md delete mode 100644 doc/release-notes/10633-add-dataverse-api-extension.md create mode 100644 doc/release-notes/6.4-release-notes.md delete mode 100644 doc/release-notes/7068-reserve-file-pids.md delete mode 100644 doc/release-notes/8129-harvesting.md delete mode 100644 doc/release-notes/8578-support-for-iso-639-3-languages.md delete mode 100644 doc/release-notes/8581-add-crossref-pid-provider.md delete mode 100644 doc/release-notes/8796-fix-license-display-indexing.md delete mode 100644 doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md delete mode 100644 doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md delete mode 100644 doc/release-notes/9317-delete-saved-search.md delete mode 100644 doc/release-notes/api-blocking-filter-json.md delete mode 100644 doc/release-notes/make-data-count-.md delete mode 100644 doc/release-notes/permalink-base-urls.md diff --git a/doc/release-notes/10169-JSON-schema-validation.md b/doc/release-notes/10169-JSON-schema-validation.md deleted file mode 100644 index 92ff4a917d5..00000000000 --- a/doc/release-notes/10169-JSON-schema-validation.md +++ /dev/null @@ -1,3 +0,0 @@ -### Improved JSON Schema validation for datasets - -Enhanced JSON schema validation with checks for required and allowed child objects, type checking for field types including `primitive`, `compound` and `controlledVocabulary`. More user-friendly error messages to help pinpoint the issues in the dataset JSON. See [Retrieve a Dataset JSON Schema for a Collection](https://guides.dataverse.org/en/6.3/api/native-api.html#retrieve-a-dataset-json-schema-for-a-collection) in the API Guide and PR #10543. diff --git a/doc/release-notes/10287-use-support-address-in-system-email-text.md b/doc/release-notes/10287-use-support-address-in-system-email-text.md deleted file mode 100644 index 4c294404298..00000000000 --- a/doc/release-notes/10287-use-support-address-in-system-email-text.md +++ /dev/null @@ -1,4 +0,0 @@ -### Notification Email Improvement - -The system email text has been improved to use the support email address (`dataverse.mail.support-email`) in the text where it states; "contact us for support at", instead of the default system email address. -Using the system email address here was particularly problematic when it was a 'noreply' address. diff --git a/doc/release-notes/10341-croissant.md b/doc/release-notes/10341-croissant.md deleted file mode 100644 index 15bc7029099..00000000000 --- a/doc/release-notes/10341-croissant.md +++ /dev/null @@ -1,9 +0,0 @@ -A new metadata export format called Croissant is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. - -When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://dataverse-guide--10533.org.readthedocs.build/en/10533/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide. - -For more about the Croissant exporter, see https://github.com/gdcc/exporter-croissant - -For installation instructions, see [Enabling External Exporters](https://dataverse-guide--10533.org.readthedocs.build/en/10533/installation/advanced.html#enabling-external-exporters) in the Installation Guide. - -See also Issue #10341 and PR #10533. diff --git a/doc/release-notes/10433-add-thumbnail-for-featured-dataverses.md b/doc/release-notes/10433-add-thumbnail-for-featured-dataverses.md deleted file mode 100644 index 0ebb84a8eb0..00000000000 --- a/doc/release-notes/10433-add-thumbnail-for-featured-dataverses.md +++ /dev/null @@ -1,5 +0,0 @@ -Add the ability to configure a thumbnail logo that is displayed for a collection when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection. - -The HTML preview of the documentation can be found [here](https://dataverse-guide--10433.org.readthedocs.build/en/10433/user/dataverse-management.html#theme). - -For more information, see [#10291](https://github.com/IQSS/dataverse/issues/10291). diff --git a/doc/release-notes/10508-base-image-fixes.md b/doc/release-notes/10508-base-image-fixes.md deleted file mode 100644 index 148066435e8..00000000000 --- a/doc/release-notes/10508-base-image-fixes.md +++ /dev/null @@ -1,12 +0,0 @@ -# Security and Compatibility Fixes to the Container Base Image - -- Switch "wait-for" to "wait4x", aligned with the Configbaker Image -- Update "jattach" to v2.2 -- Install AMD64 / ARM64 versions of tools as necessary -- Run base image as unprivileged user by default instead of `root` - this was an oversight from OpenShift changes -- Linux User, Payara Admin and Domain Master passwords: - - Print hints about default, public knowledge passwords in place for - - Enable replacing these passwords at container boot time -- Enable building with updates Temurin JRE image based on Ubuntu 24.04 LTS -- Fix entrypoint script troubles with pre- and postboot script files -- Unify location of files at CONFIG_DIR=/opt/payara/config, avoid writing to other places \ No newline at end of file diff --git a/doc/release-notes/10517-datasetType.md b/doc/release-notes/10517-datasetType.md deleted file mode 100644 index 2e3aff940c7..00000000000 --- a/doc/release-notes/10517-datasetType.md +++ /dev/null @@ -1,10 +0,0 @@ -### Initial Support for Dataset Types - -Out of the box, all datasets have the type "dataset" but superusers can add additional types. At this time the type can only be set at creation time via API. The types "dataset", "software", and "workflow" will be sent to DataCite when the dataset is published. - -For details see and #10517. Please note that this feature is highly experimental and is expected to evolve. - -Upgrade instructions --------------------- - -Update your Solr schema.xml file to pick up the "datasetType" additions and do a full reindex. diff --git a/doc/release-notes/10583-dataset-unlink-functionality-same-permission-as-link.md b/doc/release-notes/10583-dataset-unlink-functionality-same-permission-as-link.md deleted file mode 100644 index f97bd252db3..00000000000 --- a/doc/release-notes/10583-dataset-unlink-functionality-same-permission-as-link.md +++ /dev/null @@ -1,2 +0,0 @@ -New "Unlink Dataset" button has been added to the Dataset Page to allow a user to unlink a dataset from a collection that was previously linked with the "Link Dataset" button. The user must possess the same permissions needed to unlink the Dataset as they would to link the Dataset. -The [existing API](https://guides.dataverse.org/en/6.3/admin/dataverses-datasets.html#unlink-a-dataset) for unlinking datasets has been updated to no longer require superuser access. The "Publish Dataset" permission is now enough. diff --git a/doc/release-notes/10606-dataverse-in-windows-wsl.md b/doc/release-notes/10606-dataverse-in-windows-wsl.md deleted file mode 100644 index 9501d6e3090..00000000000 --- a/doc/release-notes/10606-dataverse-in-windows-wsl.md +++ /dev/null @@ -1 +0,0 @@ -New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See https://dataverse-guide--10608.org.readthedocs.build/en/10608/developers/windows.html #10606 and #10608. diff --git a/doc/release-notes/10633-add-dataverse-api-extension.md b/doc/release-notes/10633-add-dataverse-api-extension.md deleted file mode 100644 index f5d8030e8ac..00000000000 --- a/doc/release-notes/10633-add-dataverse-api-extension.md +++ /dev/null @@ -1 +0,0 @@ -The addDataverse (/api/dataverses/{identifier}) API endpoint has been extended to allow adding metadata blocks, input levels and facet ids at creation time, as the Dataverse page in create mode does in JSF. diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md new file mode 100644 index 00000000000..2648f478f92 --- /dev/null +++ b/doc/release-notes/6.4-release-notes.md @@ -0,0 +1,169 @@ +# Dataverse 6.4 + +Please note: To read these instructions in full, please go to https://github.com/IQSS/dataverse/releases/tag/v6.4 rather than the list of releases, which will cut them off. + +This release brings new features, enhancements, and bug fixes to Dataverse. Thank you to all of the community members who contributed code, suggestions, bug reports, and other assistance across the project. + +## Release Highlights + +## Features + +### Full List of ISO 639-3 Languages Now Supported + +The controlled vocabulary values list for the metadata field "Language" in the citation block has now been extended to include roughly 7920 ISO 639-3 values. + +Some of the language entries in the pre-6.4 list correspond to "macro languages" in ISO-639-3 and admins/users may wish to update to use the corresponding individual language entries from ISO-639-3. As these cases are expected to be rare (they do not involve major world languages), finding them is not covered in the release notes. Anyone who desires help in this area is encouraged to reach out to the Dataverse community via any of the standard communication channels. + +ISO 639-3 codes were downloaded from [sil.org](https://iso639-3.sil.org/code_tables/download_tables#Complete%20Code%20Tables:~:text=iso%2D639%2D3_Code_Tables_20240415.zip) and the file used for merging with the existing citation.tsv was "iso-639-3.tab". See also #8578 and #10762. + +### Unlink Dataset Button + +New "Unlink Dataset" button has been added to the dataset page to allow a user to unlink a dataset from a collection. The user must possess the same permissions needed to unlink the dataset as they would to link the Dataset. Additionally, the [existing API](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#unlink-a-dataset) for unlinking datasets has been updated to no longer require superuser access as the "Publish Dataset" permission is now enough. See also #10583 and #10689. + +### Pre-Publish File DOI Reservation + +Dataverse installations using DataCite as a persistent identifier (PID) provider (or other providers that support reserving PIDs) will be able to reserve PIDs for files when they are uploaded (rather than at publication time). Note that reserving file DOIs can slow uploads with large numbers of files so administrators may need to adjust timeouts (specifically any Apache "``ProxyPass / ajp://localhost:8009/ timeout=``" setting in the recommended Dataverse configuration). + +### Initial Support for Dataset Types + +Out of the box, all datasets now have the type "dataset" but superusers can add additional types. At this time the type of a dataset can only be set at creation time via API. The types "dataset", "software", and "workflow" (just those three, for now) will be sent to DataCite when the dataset is published. + +For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-management.html#dataset-types), #10517 and #10694. Please note that this feature is highly experimental and is expected to [evolve](https://github.com/IQSS/dataverse-pm/issues/307). + +### Collection Thumbnail Logo for Featured Collections + +Collections can now have a thumbnail logo that is displayed when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection as explained in [the guides](https://guides.dataverse.org/en/6.4/user/dataverse-management.html#theme). See also #10291 and #10433. + +### Croissant + +A new metadata export format called Croissant is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. + +When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide and #10382. + +For more about the Croissant exporter, including installation instructions, see . See also #10341, #10533, and [discussion](https://groups.google.com/g/dataverse-community/c/JI8HPgGarr8/m/DqEIkiwlAgAJ) on the mailing list. + +### Saved Searches Can Be Deleted + +Saved searches can now be deleted via API. See the [Saved Search](https://guides.dataverse.org/en/6.4/api/native-api.html#saved-search) section of the API Guide, #9317 and #10198. + +### Notification Email Improvement + +When notification emails are sent the part of the closing that says "contact us for support at" will now show the support email address (`dataverse.mail.support-email`), when configured, instead of the default system email address. Using the system email address here was particularly problematic when it was a "noreply" address. See also #10287 and #10504. + +### More Flexible PermaLinks + +The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to support greater flexibility. Previously, the string `/citation?persistentId=` was automatically appended to the configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic additions. See also #10775. + +### Development on Windows + +New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See [the guides](https://guides.dataverse.org/en/6.4/developers/windows.html), #10606, and #10608. + +### Experimental Crossref PID (DOI) Provider + +Crossref can now be used as a PID (DOI) provider, but this feature is experimental. Please provide feedback through the usual channels. See also the [guides](https://guides.dataverse.org/en/6.4/installation/config.html#crossref-specific-settings), #8581, and #10806. + +### Improved JSON Schema Validation for Datasets + +JSON Schema validation has been enhanced with checks for required and allowed child objects as well as type checking for field types including `primitive`, `compound` and `controlledVocabulary`. More user-friendly error messages help pinpoint the issues in the dataset JSON. See [Retrieve a Dataset JSON Schema for a Collection](https://guides.dataverse.org/en/6.4/api/native-api.html#retrieve-a-dataset-json-schema-for-a-collection) in the API Guide, #10169, and #10543. + +### Counter Processor 1.05 Support (Make Data Count) + +Counter Processor 1.05 is now supported for use with Make Data Count. If you are running Counter Processor, you should reinstall/reconfigure it as described in the latest guides. Note that Counter Processor 1.05 requires Python 3, so you will need to follow the full Counter Processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse. See also #10479. + +## Bug Fixes + +### No License Nor Terms of Use + +When datasets have neither a license nor custom terms of use, the dataset page will indicate this. Also, these datasets will no longer be indexed as having custom terms. See also #8796, #10513, and #10614. + +### CC0 License Bug Fix + +At a high level, some datasets have been mislabeled as "Custom License" when they should have been "CC0 1.0". This has been corrected. + +In Dataverse 5.10, datasets with only "CC0 Waiver" in the "termsofuse" field were converted to "Custom License" (instead of the CC0 1.0 license) through a SQL migration script (see #10634). On deployment of Dataverse 6.4, a new SQL migration script will be run automatically to correct this, changing these datasets to CC0. You can review the script in #10634, which only affect the following datasets: + +- The existing "Terms of Use" must be equal to "This dataset is made available under a Creative Commons CC0 license with the following additional/modified terms and conditions: CC0 Waiver" (this was set in #10634). +- The following terms fields must be empty: Confidentiality Declaration, Special Permissions, Restrictions, Citation Requirements, Depositor Requirements, Conditions, and Disclaimer. +- The license ID must not be assigned. + +The script will set the license ID to that of the CC0 1.0 license and remove the contents of "termsofuse" field. See also #9081 and #10634. + +### Remap oai_dc Export and Harvesting Format Fields: dc:type and dc:date + +The `oai_dc` export and harvesting format has had the following fields remapped: + +- dc:type was mapped to the field "Kind of Data". Now it is hard-coded to the word "Dataset". +- dc:date was mapped to the field "Production Date" when available and otherwise to "Publication Date". Now it is mapped the field "Publication Date" or the field used for the citation date, if set (see [Set Citation Date Field Type for a Dataset](https://guides.dataverse.org/en/6.4/api/native-api.html#set-citation-date-field-type-for-a-dataset)). + +In order for these changes to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) should be run (mentioned below). See #8129 and #10737. + +### Zip File No Longer Misdetected as Shapefile (Hidden Directories) + +When detecting files types, Dataverse would previously detect a zip file as a shapefile if it contained [markers of a shapefile](https://guides.dataverse.org/en/6.4/developers/geospatial.html) in hidden directories. These hidden directories are now ignored when deciding if a zip file is a shapefile or not. See also #8945 and #10627. + +### Valid JSON in Error Response + +When any `ApiBlockingFilter` policy applies to a request, the JSON in the body of the error response is now valid JSON. See also #10085. + +### Docker Container Base Image Security and Compatibility + +- Switch "wait-for" to "wait4x", aligned with the Configbaker Image +- Update "jattach" to v2.2 +- Install AMD64 / ARM64 versions of tools as necessary +- Run base image as unprivileged user by default instead of `root` - this was an oversight from OpenShift changes +- Linux User, Payara Admin and Domain Master passwords: + - Print hints about default, public knowledge passwords in place for + - Enable replacing these passwords at container boot time +- Enable building with updates Temurin JRE image based on Ubuntu 24.04 LTS +- Fix entrypoint script troubles with pre- and postboot script files +- Unify location of files at CONFIG_DIR=/opt/payara/config, avoid writing to other places + +See also #10508, #10672 and #10722. + +## API + +### addDataverse + +The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended to allow adding metadata blocks, input levels and facet IDs at creation time, as the Dataverse page in create mode does in JSF. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#create-a-dataverse-collection) #10633 and #10644. + +## Major Use Cases + +- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) + +## Settings + +The following settings have been added: + +- dataverse.pid.*.crossref.url +- dataverse.pid.*.crossref.rest-api-url +- dataverse.pid.*.crossref.username +- dataverse.pid.*.crossref.password +- dataverse.pid.*.crossref.depositor +- dataverse.pid.*.crossref.depositor-email + +### Backward incompatible changes + +The oai_dc export format has changed. See the "Remap oai_dc" section above. + +## Complete List of Changes + +## Getting Help + +## Upgrade Instructions + +In order for changes to the `oai_dc` metadata export format to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) should be run. See also "remap oai_dc" above. + +Update your Solr schema.xml file to pick up the "dataset type" additions described above and do a full reindex. + +### Additional Upgrade Steps + +- If you currently use a PermaLink provider with a configured `base-url`: You must manually append + "/citation?persistentId=" to the existing base URL to maintain functionality. +- If you use a PermaLink provider without a configured `base-url`: No changes are required. + +6\. Update the Citation metadata block: + +``` +- `wget https://github.com/IQSS/dataverse/releases/download/v6.4/citation.tsv` +- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` +``` diff --git a/doc/release-notes/7068-reserve-file-pids.md b/doc/release-notes/7068-reserve-file-pids.md deleted file mode 100644 index 182a0d7f67b..00000000000 --- a/doc/release-notes/7068-reserve-file-pids.md +++ /dev/null @@ -1,9 +0,0 @@ -## Release Highlights - -### Pre-Publish File DOI Reservation with DataCite - -Dataverse installations using DataCite (or other persistent identifier (PID) Providers that support reserving PIDs) will be able to reserve PIDs for files when they are uploaded (rather than at publication time). Note that reserving file DOIs can slow uploads with large numbers of files so administrators may need to adjust timeouts (specifically any Apache "``ProxyPass / ajp://localhost:8009/ timeout=``" setting in the recommended Dataverse configuration). - -## Major Use Cases - -- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) diff --git a/doc/release-notes/8129-harvesting.md b/doc/release-notes/8129-harvesting.md deleted file mode 100644 index 63ca8744941..00000000000 --- a/doc/release-notes/8129-harvesting.md +++ /dev/null @@ -1,18 +0,0 @@ -### Remap oai_dc export and harvesting format fields: dc:type and dc:date - -The `oai_dc` export and harvesting format has had the following fields remapped: - -- dc:type was mapped to the field "Kind of Data". Now it is hard-coded to the word "Dataset". -- dc:date was mapped to the field "Production Date" when available and otherwise to "Publication Date". Now it is mapped the field "Publication Date" or the field used for the citation date, if set (see [Set Citation Date Field Type for a Dataset](https://guides.dataverse.org/en/6.3/api/native-api.html#set-citation-date-field-type-for-a-dataset)). - -In order for these changes to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.3/admin/metadataexport.html#batch-exports-through-the-api) should be run. - -For more information, please see #8129 and #10737. - -### Backward incompatible changes - -See the "Remap oai_dc export" section above. - -### Upgrade instructions - -In order for changes to the `oai_dc` metadata export format to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.3/admin/metadataexport.html#batch-exports-through-the-api) should be run. diff --git a/doc/release-notes/8578-support-for-iso-639-3-languages.md b/doc/release-notes/8578-support-for-iso-639-3-languages.md deleted file mode 100644 index c702b6b8a59..00000000000 --- a/doc/release-notes/8578-support-for-iso-639-3-languages.md +++ /dev/null @@ -1,17 +0,0 @@ -The Controlled Vocabulary Values list for the metadata field Language in the Citation block has now been extended to include roughly 7920 ISO 639-3 values. -- Some of the language entries in the pre-v.6.4 list correspond to "macro languages" in ISO-639-3 and admins/users may wish to update to use the corresponding individual language entries from ISO-639-3. As these cases are expected to be rare (they do not involve major world languages), finding them is not covered in the release notes. Anyone who desires help in this area is encouraged to reach out to the Dataverse community via any of the standard communication channels. -- ISO 639-3 codes were downloaded from: -``` -https://iso639-3.sil.org/code_tables/download_tables#Complete%20Code%20Tables:~:text=iso%2D639%2D3_Code_Tables_20240415.zip -``` -- The file used for merging with the existing citation.tsv was iso-639-3.tab - -To be added to the 6.4 release instructions: - -### Additional Upgrade Steps -6\. Update the Citation metadata block: - -``` -- `wget https://github.com/IQSS/dataverse/releases/download/v6.4/citation.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` -``` diff --git a/doc/release-notes/8581-add-crossref-pid-provider.md b/doc/release-notes/8581-add-crossref-pid-provider.md deleted file mode 100644 index 3610aa6d7cc..00000000000 --- a/doc/release-notes/8581-add-crossref-pid-provider.md +++ /dev/null @@ -1,3 +0,0 @@ -Added CrossRef DOI Pid Provider - -See Installation Configuration document for JVM Settings to enable CrossRef as a Pid Provider diff --git a/doc/release-notes/8796-fix-license-display-indexing.md b/doc/release-notes/8796-fix-license-display-indexing.md deleted file mode 100644 index ebded088875..00000000000 --- a/doc/release-notes/8796-fix-license-display-indexing.md +++ /dev/null @@ -1 +0,0 @@ -When datasets have neither a license nor custom terms of use the display will indicate this. Also, these datasets will no longer be indexed as having custom terms. diff --git a/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md b/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md deleted file mode 100644 index 145ae5f6d55..00000000000 --- a/doc/release-notes/8945-ignore-shapefiles-under-hidden-directories-in-zip.md +++ /dev/null @@ -1,5 +0,0 @@ -### Shapefile Handling will now ignore files under a hidden directory within the zip file - -Directories that are hidden will be ignored when determining if a zip file contains Shapefile files. - -For more information, see #8945. \ No newline at end of file diff --git a/doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md b/doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md deleted file mode 100644 index 042b2ec39fd..00000000000 --- a/doc/release-notes/9081-CC0-waiver-turned-into-custom-license.md +++ /dev/null @@ -1,6 +0,0 @@ -In an earlier Dataverse release, Datasets with only 'CC0 Waiver' in termsofuse field were converted to 'Custom License' instead of CC0 1.0 licenses during an automated process. A new process was added to correct this. Only Datasets with no terms other than the one create by the previous process will be modified. -- The existing 'Terms of Use' must be equal to 'This dataset is made available under a Creative Commons CC0 license with the following additional/modified terms and conditions: CC0 Waiver' -- The following terms fields must be empty: Confidentiality Declaration, Special Permissions, Restrictions, Citation Requirements, Depositor Requirements, Conditions, and Disclaimer. -- The License ID must not be assigned. - -This process will set the License ID to that of the CC0 1.0 license and remove the contents of termsofuse field. diff --git a/doc/release-notes/9317-delete-saved-search.md b/doc/release-notes/9317-delete-saved-search.md deleted file mode 100644 index 34723801036..00000000000 --- a/doc/release-notes/9317-delete-saved-search.md +++ /dev/null @@ -1,4 +0,0 @@ -### Saved search deletion - -Saved searches can now be removed using API `/api/admin/savedsearches/$id`. See PR #10198. -This is reflected in the [Saved Search Native API section](https://dataverse-guide--10198.org.readthedocs.build/en/10198/api/native-api.html#saved-search) of the Guide. \ No newline at end of file diff --git a/doc/release-notes/api-blocking-filter-json.md b/doc/release-notes/api-blocking-filter-json.md deleted file mode 100644 index 337ff82dd8b..00000000000 --- a/doc/release-notes/api-blocking-filter-json.md +++ /dev/null @@ -1,3 +0,0 @@ -* When any `ApiBlockingFilter` policy applies to a request, the JSON in the body of the error response is now valid JSON. - In case an API client did any special processing to allow it to parse the body, that is no longer necessary. - The status code of such responses has not changed. diff --git a/doc/release-notes/make-data-count-.md b/doc/release-notes/make-data-count-.md deleted file mode 100644 index 9022582dddb..00000000000 --- a/doc/release-notes/make-data-count-.md +++ /dev/null @@ -1,3 +0,0 @@ -### Counter Processor 1.05 Support - -This release includes support for counter-processor-1.05 for processing Make Data Count metrics. If you are running Make Data Counts support, you should reinstall/reconfigure counter-processor as described in the latest Guides. (For existing installations, note that counter-processor-1.05 requires a Python3, so you will need to follow the full counter-processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse.) diff --git a/doc/release-notes/permalink-base-urls.md b/doc/release-notes/permalink-base-urls.md deleted file mode 100644 index 1dd74057351..00000000000 --- a/doc/release-notes/permalink-base-urls.md +++ /dev/null @@ -1,10 +0,0 @@ -The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to -support greater flexibility. Previously, the string "/citation?persistentId=" was automatically appended to the -configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic -additions. - -**Upgrade instructions:** - -- If you currently use a PermaLink provider with a configured `base-url`: You must manually append - "/citation?persistentId=" to the existing base URL to maintain functionality. -- If you use a PermaLink provider without a configured `base-url`: No changes are required. \ No newline at end of file From c69e78337c6073649928720a3ee1ad6703687847 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Sep 2024 15:12:51 -0400 Subject: [PATCH 07/44] add second half of snippets #10853 --- .../10726-dataverse-facets-api-extension.md | 3 - ...ublication-status-to-search-api-results.md | 14 -- ...st-metadatablocks-display-on-create-fix.md | 1 - doc/release-notes/10744-ro-crate-docs.md | 3 - ...ataverse-user-permissions-api-extension.md | 1 - doc/release-notes/10758-rust-client.md | 3 - .../10797-update-current-version-bug-fix.md | 11 -- .../10800-add-dataverse-request-json-fix.md | 1 - .../10810-search-api-payload-extensions.md | 52 ------ .../10819-publish-thumbnail-bug.md | 6 - doc/release-notes/6.4-release-notes.md | 153 ++++++++++++++++-- 11 files changed, 144 insertions(+), 104 deletions(-) delete mode 100644 doc/release-notes/10726-dataverse-facets-api-extension.md delete mode 100644 doc/release-notes/10733-add-publication-status-to-search-api-results.md delete mode 100644 doc/release-notes/10741-list-metadatablocks-display-on-create-fix.md delete mode 100644 doc/release-notes/10744-ro-crate-docs.md delete mode 100644 doc/release-notes/10749-dataverse-user-permissions-api-extension.md delete mode 100644 doc/release-notes/10758-rust-client.md delete mode 100644 doc/release-notes/10797-update-current-version-bug-fix.md delete mode 100644 doc/release-notes/10800-add-dataverse-request-json-fix.md delete mode 100644 doc/release-notes/10810-search-api-payload-extensions.md delete mode 100644 doc/release-notes/10819-publish-thumbnail-bug.md diff --git a/doc/release-notes/10726-dataverse-facets-api-extension.md b/doc/release-notes/10726-dataverse-facets-api-extension.md deleted file mode 100644 index baf6f798e35..00000000000 --- a/doc/release-notes/10726-dataverse-facets-api-extension.md +++ /dev/null @@ -1,3 +0,0 @@ -New optional query parameter "returnDetails" added to "dataverses/{identifier}/facets/" endpoint to include detailed information of each DataverseFacet. - -New endpoint "datasetfields/facetables" that lists all facetable dataset fields defined in the installation. diff --git a/doc/release-notes/10733-add-publication-status-to-search-api-results.md b/doc/release-notes/10733-add-publication-status-to-search-api-results.md deleted file mode 100644 index d015a50a00d..00000000000 --- a/doc/release-notes/10733-add-publication-status-to-search-api-results.md +++ /dev/null @@ -1,14 +0,0 @@ -Search API (/api/search) response will now include publicationStatuses in the Json response as long as the list is not empty - -Example: -```javascript -"items": [ - { - "name": "Darwin's Finches", - ... - "publicationStatuses": [ - "Unpublished", - "Draft" - ], -(etc, etc) -``` diff --git a/doc/release-notes/10741-list-metadatablocks-display-on-create-fix.md b/doc/release-notes/10741-list-metadatablocks-display-on-create-fix.md deleted file mode 100644 index 4edadcaa1fc..00000000000 --- a/doc/release-notes/10741-list-metadatablocks-display-on-create-fix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed dataverses/{identifier}/metadatablocks endpoint to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. diff --git a/doc/release-notes/10744-ro-crate-docs.md b/doc/release-notes/10744-ro-crate-docs.md deleted file mode 100644 index 9d52b4578b4..00000000000 --- a/doc/release-notes/10744-ro-crate-docs.md +++ /dev/null @@ -1,3 +0,0 @@ -## RO-Crate Support (Metadata Export) - -Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) in the sense that dataset metadata can be exported in that format. This functionality is not available out of the box but you can enable one or more RO-Crate exporters from the [list of external exporters](https://preview.guides.gdcc.io/en/develop/installation/advanced.html#inventory-of-external-exporters). See also #10744. diff --git a/doc/release-notes/10749-dataverse-user-permissions-api-extension.md b/doc/release-notes/10749-dataverse-user-permissions-api-extension.md deleted file mode 100644 index 706b1f42641..00000000000 --- a/doc/release-notes/10749-dataverse-user-permissions-api-extension.md +++ /dev/null @@ -1 +0,0 @@ -New API endpoint "dataverses/{identifier}/userPermissions" for obtaining the user permissions on a dataverse. diff --git a/doc/release-notes/10758-rust-client.md b/doc/release-notes/10758-rust-client.md deleted file mode 100644 index e206f27ce65..00000000000 --- a/doc/release-notes/10758-rust-client.md +++ /dev/null @@ -1,3 +0,0 @@ -### Rust API client library - -An API client library for the Rust programming language is now available at https://github.com/gdcc/rust-dataverse and has been added to the [list of client libraries](https://dataverse-guide--10758.org.readthedocs.build/en/10758/api/client-libraries.html) in the API Guide. See also #10758. diff --git a/doc/release-notes/10797-update-current-version-bug-fix.md b/doc/release-notes/10797-update-current-version-bug-fix.md deleted file mode 100644 index 2cfaf69cad3..00000000000 --- a/doc/release-notes/10797-update-current-version-bug-fix.md +++ /dev/null @@ -1,11 +0,0 @@ -A significant bug in the superuser-only "Update-Current-Version" publication was found and fixed in this release. If the Update-Current-Version option was used when changes were made to the dataset Terms (rather than to dataset metadata), or if the PID provider service was down/returned an error, the update would fail and render the dataset unusable and require restoration from a backup. The fix in this release allows the update to succeed in both of these cases and redesigns the functionality such that any unknown issues should not make the dataset unusable (i.e. the error would be reported and the dataset would remain in its current state with the last-published version as it was and changes still in the draft version.) - -Users of earlier Dataverse releases are encouraged to alert their superusers to this issue. Those who wish to disable this functionality have two options: -* Change the dataset.updateRelease entry in the Bundle.properties file (or local language version) to "Do Not Use" or similar (doesn't disable but alerts superusers to the issue), or -* Edit the dataset.xhtml file to remove the lines - - - - - -, delete the contents of the generated and osgi-cache directories in the Dataverse Payara domain, and restart the Payara server. diff --git a/doc/release-notes/10800-add-dataverse-request-json-fix.md b/doc/release-notes/10800-add-dataverse-request-json-fix.md deleted file mode 100644 index ddd6c388ec6..00000000000 --- a/doc/release-notes/10800-add-dataverse-request-json-fix.md +++ /dev/null @@ -1 +0,0 @@ -Fixed the "addDataverse" API endpoint (/dataverses/{id} POST) expected request JSON structure to parse facetIds as described in the docs. \ No newline at end of file diff --git a/doc/release-notes/10810-search-api-payload-extensions.md b/doc/release-notes/10810-search-api-payload-extensions.md deleted file mode 100644 index 5112d9f62ee..00000000000 --- a/doc/release-notes/10810-search-api-payload-extensions.md +++ /dev/null @@ -1,52 +0,0 @@ -Search API (/api/search) response will now include new fields for the different entities. - -For Dataverse: - -- "affiliation" -- "parentDataverseName" -- "parentDataverseIdentifier" -- "image_url" (optional) - -```javascript -"items": [ - { - "name": "Darwin's Finches", - ... - "affiliation": "Dataverse.org", - "parentDataverseName": "Root", - "parentDataverseIdentifier": "root", - "image_url":"data:image/png;base64,iVBORw0..." -(etc, etc) -``` - -For DataFile: - -- "releaseOrCreateDate" -- "image_url" (optional) - -```javascript -"items": [ - { - "name": "test.txt", - ... - "releaseOrCreateDate": "2016-05-10T12:53:39Z", - "image_url":"data:image/png;base64,iVBORw0..." -(etc, etc) -``` - -For Dataset: - -- "image_url" (optional) - -```javascript -"items": [ - { - ... - "image_url": "http://localhost:8080/api/datasets/2/logo" - ... -(etc, etc) -``` - -The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn’t returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF. Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. - -The schema.xml file for Solr has been updated to include a new field called dvParentAlias for supporting the new response field "parentDataverseIdentifier". So for the next Dataverse released version, a Solr reindex will be necessary to apply the new schema.xml version. diff --git a/doc/release-notes/10819-publish-thumbnail-bug.md b/doc/release-notes/10819-publish-thumbnail-bug.md deleted file mode 100644 index 46c9875a6ef..00000000000 --- a/doc/release-notes/10819-publish-thumbnail-bug.md +++ /dev/null @@ -1,6 +0,0 @@ -The initial release of the Dataverse v6.3 introduced a bug where publishing would break the dataset thumbnail, which in turn broke the rendering of the parent Collection ("dataverse") page. This problem was fixed in the PR 10820. - -This bug fix will prevent this from happening in the future, but does not fix any existing broken links. To restore any broken thumbnails caused by this bug, you can call the http://localhost:8080/api/admin/clearThumbnailFailureFlag API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the http://localhost:8080/api/admin/clearThumbnailFailureFlag/id to clear the flag for individual files. Calling the former, batch API is recommended. - -Additionally, the same PR made it possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by raising the feature flag `-Ddataverse.feature.disable-dataset-thumbnail-autoselect=true`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. - diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 2648f478f92..feb8e54102a 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -6,6 +6,8 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ## Release Highlights +- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) + ## Features ### Full List of ISO 639-3 Languages Now Supported @@ -34,14 +36,22 @@ For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-ma Collections can now have a thumbnail logo that is displayed when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection as explained in [the guides](https://guides.dataverse.org/en/6.4/user/dataverse-management.html#theme). See also #10291 and #10433. -### Croissant +### Croissant Support (Metadata Export) -A new metadata export format called Croissant is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. +A new metadata export format called [Croissant](https://github.com/mlcommons/croissant) is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. -When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide and #10382. +When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide. This change was made in Dataverse 6.3 in #10382. For more about the Croissant exporter, including installation instructions, see . See also #10341, #10533, and [discussion](https://groups.google.com/g/dataverse-community/c/JI8HPgGarr8/m/DqEIkiwlAgAJ) on the mailing list. +### RO-Crate Support (Metadata Export) + +Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) as a metadata export format. This functionality is not available out of the box, but you can enable one or more RO-Crate exporters from the [list of external exporters](https://guides.dataverse.org/en/6.4/installation/advanced.html#inventory-of-external-exporters). See also #10744 and #10796. + +### Rust API Client Library + +An Dataverse API client library for the Rust programming language is now available at https://github.com/gdcc/rust-dataverse and has been added to the [list of client libraries](https://guides.dataverse.org/en/6.4/api/client-libraries.html) in the API Guide. See also #10758. + ### Saved Searches Can Be Deleted Saved searches can now be deleted via API. See the [Saved Search](https://guides.dataverse.org/en/6.4/api/native-api.html#saved-search) section of the API Guide, #9317 and #10198. @@ -50,6 +60,10 @@ Saved searches can now be deleted via API. See the [Saved Search](https://guides When notification emails are sent the part of the closing that says "contact us for support at" will now show the support email address (`dataverse.mail.support-email`), when configured, instead of the default system email address. Using the system email address here was particularly problematic when it was a "noreply" address. See also #10287 and #10504. +### Ability to Disable Automatic Thumbnail Selection + +It is now possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by enabling the [feature flag](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) `dataverse.feature.disable-dataset-thumbnail-autoselect`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. See also #10820. + ### More Flexible PermaLinks The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to support greater flexibility. Previously, the string `/citation?persistentId=` was automatically appended to the configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic additions. See also #10775. @@ -72,7 +86,34 @@ Counter Processor 1.05 is now supported for use with Make Data Count. If you are ## Bug Fixes -### No License Nor Terms of Use +### Update Current Version + +A significant bug in the superuser-only [Update Current Version](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#make-metadata-updates-without-changing-dataset-version) publication option was fixed. If the "Update Current Version" option was used when changes were made to the dataset terms (rather than to dataset metadata) or if the PID provider service was down or returned an error, the update would fail and render the dataset unusable and require restoration from a backup. The fix in this release allows the update to succeed in both of these cases and redesigns the functionality such that any unknown issues should not make the dataset unusable (i.e. the error would be reported and the dataset would remain in its current state with the last-published version as it was and changes still in the draft version.) + +If you do not plan to upgrade to Dataverse 6.4 right away, you are encouraged to alert your superusers to this issue (see [this post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ)). Here are some workarounds for pre-6.4 versions: + +* Change the "dataset.updateRelease" entry in the Bundle.properties file (or local language version) to "Do Not Use" or similar (this doesn't disable the button but alerts superusers to the issue), or +* Edit the dataset.xhtml file to remove the lines below, delete the contents of the generated and osgi-cache directories in the Dataverse Payara domain, and restart the Payara server. This will remove the "Update Current Version" from the UI. + +``` + + + +``` + +Again, the workarounds above are only for pre-6.4 versions. The bug has been fixed in Dataverse 6.4. See also #10797. + +### Broken Thumbnails + +Dataverse 6.3 introduced a bug where publishing would break the dataset thumbnail, which in turn broke the rendering of the parent collection (dataverse) page. + +This bug has been fixed but any existing broken thumbnails must be fixed manually. See "clearThumbnailFailureFlag" in the upgrade instructions below. + +Additionally, it is now possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by raising the feature flag `-Ddataverse.feature.disable-dataset-thumbnail-autoselect=true`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. + +See also #10819, #10820, and [the post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list. + +### No License, No Terms of Use When datasets have neither a license nor custom terms of use, the dataset page will indicate this. Also, these datasets will no longer be indexed as having custom terms. See also #8796, #10513, and #10614. @@ -122,18 +163,108 @@ See also #10508, #10672 and #10722. ## API -### addDataverse +### Search API: affiliation, parentDataverseName, image_url, etc. -The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended to allow adding metadata blocks, input levels and facet IDs at creation time, as the Dataverse page in create mode does in JSF. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#create-a-dataverse-collection) #10633 and #10644. +The Search API (`/api/search`) response now includes additional fields, depending on the type. -## Major Use Cases +For collections (dataverses): -- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) +- "affiliation" +- "parentDataverseName" +- "parentDataverseIdentifier" +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "Darwin's Finches", + ... + "affiliation": "Dataverse.org", + "parentDataverseName": "Root", + "parentDataverseIdentifier": "root", + "image_url":"data:image/png;base64,iVBORw0..." +(etc, etc) +``` + +For datasets: + +- "image_url" (optional) + +```javascript +"items": [ + { + ... + "image_url": "http://localhost:8080/api/datasets/2/logo" + ... +(etc, etc) +``` + +For files: + +- "releaseOrCreateDate" +- "image_url" (optional) + +```javascript +"items": [ + { + "name": "test.txt", + ... + "releaseOrCreateDate": "2016-05-10T12:53:39Z", + "image_url":"data:image/png;base64,iVBORw0..." +(etc, etc) +``` + +These examples are also shown in the [Search API](https://guides.dataverse.org/en/6.4/api/search.html) section of the API Guide. + +The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn't returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF. Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. + +The Solr schema.xml file has been updated to include a new field called "dvParentAlias" for supporting the new response field "parentDataverseIdentifier". See upgrade instructions below. + +See also #10810 and #10811. + +### Search API: publicationStatuses + +The Search API (`/api/search`) response will now include publicationStatuses in the JSON response as long as the list is not empty. + +Example: + +```javascript +"items": [ + { + "name": "Darwin's Finches", + ... + "publicationStatuses": [ + "Unpublished", + "Draft" + ], +(etc, etc) +``` + +See also #10733 and #10738. + +### Search Facet Information Exposed + +A new endpoint `/api/datasetfields/facetables` lists all facetable dataset fields defined in the installation, as described in [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-facetable-dataset-fields). + +A new optional query parameter "returnDetails" added to `/api/dataverses/{identifier}/facets/` endpoint to include detailed information of each DataverseFacet, as described in [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-facets-configured-for-a-dataverse-collection). See also #10726 and #10727. + +### User Permissions on Collections + +A new endpoint at `/api/dataverses/{identifier}/userPermissions` for obtaining the user permissions on a collection (dataverse) has been added. See also #10749 and #10751. + +### addDataverse Extended + +The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended to allow adding metadata blocks, input levels and facet IDs at creation time, as the Dataverse page in create mode does in JSF. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#create-a-dataverse-collection), #10633 and #10644. + +### Metadata Blocks and Display on Create + +The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. ## Settings The following settings have been added: +- dataverse.feature.disable-dataset-thumbnail-autoselect - dataverse.pid.*.crossref.url - dataverse.pid.*.crossref.rest-api-url - dataverse.pid.*.crossref.username @@ -153,7 +284,7 @@ The oai_dc export format has changed. See the "Remap oai_dc" section above. In order for changes to the `oai_dc` metadata export format to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) should be run. See also "remap oai_dc" above. -Update your Solr schema.xml file to pick up the "dataset type" additions described above and do a full reindex. +Update your Solr schema.xml file to pick up the "datasetType" and "dvParentAlias" additions described above and do a full reindex. ### Additional Upgrade Steps @@ -161,6 +292,10 @@ Update your Solr schema.xml file to pick up the "dataset type" additions describ "/citation?persistentId=" to the existing base URL to maintain functionality. - If you use a PermaLink provider without a configured `base-url`: No changes are required. +To restore any broken thumbnails caused by this bug, you can call the http://localhost:8080/api/admin/clearThumbnailFailureFlag API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the http://localhost:8080/api/admin/clearThumbnailFailureFlag/id to clear the flag for individual files. Calling the former, batch API is recommended. + + + 6\. Update the Citation metadata block: ``` From c706372fd994df2bbe293949059e21f586d9c81d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 19 Sep 2024 16:49:45 -0400 Subject: [PATCH 08/44] add highlights and upgrade steps #10853 --- doc/release-notes/6.4-release-notes.md | 198 ++++++++++++++++++++++--- 1 file changed, 178 insertions(+), 20 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index feb8e54102a..f6013cbe2d6 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -6,9 +6,24 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ## Release Highlights -- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time. (Issue #7068, PR #7334) +Dataverse 6.4 fixes a couple important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: -## Features +- "Update Current Version" can cause metadata loss +- Publishing breaks designated dataset thumbnail, messes up collection page + +New features include: + +- All ISO 639-3 languages are now supported +- There is now a button for "Unlink Dataset" +- Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time +- Datasets can now have types such as "software" or "workflow" +- Croissant support +- RO-Crate support +- A new Rust library + +All this and more can be found below. Read on! + +## Features Added ### Full List of ISO 639-3 Languages Now Supported @@ -32,10 +47,6 @@ Out of the box, all datasets now have the type "dataset" but superusers can add For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-management.html#dataset-types), #10517 and #10694. Please note that this feature is highly experimental and is expected to [evolve](https://github.com/IQSS/dataverse-pm/issues/307). -### Collection Thumbnail Logo for Featured Collections - -Collections can now have a thumbnail logo that is displayed when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection as explained in [the guides](https://guides.dataverse.org/en/6.4/user/dataverse-management.html#theme). See also #10291 and #10433. - ### Croissant Support (Metadata Export) A new metadata export format called [Croissant](https://github.com/mlcommons/croissant) is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. @@ -52,6 +63,10 @@ Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) as a An Dataverse API client library for the Rust programming language is now available at https://github.com/gdcc/rust-dataverse and has been added to the [list of client libraries](https://guides.dataverse.org/en/6.4/api/client-libraries.html) in the API Guide. See also #10758. +### Collection Thumbnail Logo for Featured Collections + +Collections can now have a thumbnail logo that is displayed when the collection is configured as a featured collection. If present, this thumbnail logo is shown. Otherwise, the collection logo is shown. Configuration is done under the "Theme" for a collection as explained in [the guides](https://guides.dataverse.org/en/6.4/user/dataverse-management.html#theme). See also #10291 and #10433. + ### Saved Searches Can Be Deleted Saved searches can now be deleted via API. See the [Saved Search](https://guides.dataverse.org/en/6.4/api/native-api.html#saved-search) section of the API Guide, #9317 and #10198. @@ -84,7 +99,7 @@ JSON Schema validation has been enhanced with checks for required and allowed ch Counter Processor 1.05 is now supported for use with Make Data Count. If you are running Counter Processor, you should reinstall/reconfigure it as described in the latest guides. Note that Counter Processor 1.05 requires Python 3, so you will need to follow the full Counter Processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse. See also #10479. -## Bug Fixes +## Bugs Fixed ### Update Current Version @@ -161,7 +176,7 @@ When any `ApiBlockingFilter` policy applies to a request, the JSON in the body o See also #10508, #10672 and #10722. -## API +## API Updates ### Search API: affiliation, parentDataverseName, image_url, etc. @@ -260,7 +275,7 @@ The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. -## Settings +## Settings Added The following settings have been added: @@ -272,33 +287,176 @@ The following settings have been added: - dataverse.pid.*.crossref.depositor - dataverse.pid.*.crossref.depositor-email -### Backward incompatible changes +## Backward Incompatible Changes The oai_dc export format has changed. See the "Remap oai_dc" section above. ## Complete List of Changes +For the complete list of code changes in this release, see the [6.4 milestone](https://github.com/IQSS/dataverse/issues?q=milestone%3A6.4+is%3Aclosed) in GitHub. + ## Getting Help +For help with upgrading, installing, or general questions please post to the [Dataverse Community Google Group](https://groups.google.com/g/dataverse-community) or email support@dataverse.org. + +## Installation + +If this is a new installation, please follow our [Installation Guide](https://guides.dataverse.org/en/latest/installation/). Please don't be shy about [asking for help](https://guides.dataverse.org/en/latest/installation/intro.html#getting-help) if you need it! + +Once you are in production, we would be delighted to update our [map of Dataverse installations](https://dataverse.org/installations) around the world to include yours! Please [create an issue](https://github.com/IQSS/dataverse-installations/issues) or email us at support@dataverse.org to join the club! + +You are also very welcome to join the [Global Dataverse Community Consortium](https://www.gdcc.io/) (GDCC). + ## Upgrade Instructions -In order for changes to the `oai_dc` metadata export format to be reflected in existing datasets, a [reexport all](https://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api) should be run. See also "remap oai_dc" above. +Upgrading requires a maintenance window and downtime. Please plan accordingly, create backups of your database, etc. -Update your Solr schema.xml file to pick up the "datasetType" and "dvParentAlias" additions described above and do a full reindex. +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.2. -### Additional Upgrade Steps +0\. These instructions assume that you are upgrading from the immediate previous version. If you are running an earlier version, the only supported way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to this version. -- If you currently use a PermaLink provider with a configured `base-url`: You must manually append - "/citation?persistentId=" to the existing base URL to maintain functionality. -- If you use a PermaLink provider without a configured `base-url`: No changes are required. +If you are running Payara as a non-root user (and you should be!), **remember not to execute the commands below as root**. Use `sudo` to change to that user first. For example, `sudo -i -u dataverse` if `dataverse` is your dedicated application user. -To restore any broken thumbnails caused by this bug, you can call the http://localhost:8080/api/admin/clearThumbnailFailureFlag API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the http://localhost:8080/api/admin/clearThumbnailFailureFlag/id to clear the flag for individual files. Calling the former, batch API is recommended. +In the following commands, we assume that Payara 6 is installed in `/usr/local/payara6`. If not, adjust as needed. +```shell +export PAYARA=/usr/local/payara6` +``` +(or `setenv PAYARA /usr/local/payara6` if you are using a `csh`-like shell) -6\. Update the Citation metadata block: +1\. Undeploy the previous version +```shell +$PAYARA/bin/asadmin undeploy dataverse-6.3 ``` -- `wget https://github.com/IQSS/dataverse/releases/download/v6.4/citation.tsv` -- `curl http://localhost:8080/api/admin/datasetfield/load -X POST --data-binary @citation.tsv -H "Content-type: text/tab-separated-values"` + +2\. Stop and start Payara + +```shell +service payara stop +sudo service payara start +``` + +3\. Deploy this version + +```shell +$PAYARA/bin/asadmin deploy dataverse-6.4.war +``` + +Note: if you have any trouble deploying, stop Payara, remove the following directories, start Payara, and try to deploy again. + +```shell +service payara stop +rm -rf $PAYARA/glassfish/domains/domain1/generated +rm -rf $PAYARA/glassfish/domains/domain1/osgi-cache +rm -rf $PAYARA/glassfish/domains/domain1/lib/databases +``` + +4\. For installations with internationalization: + +Please remember to update translations via [Dataverse language packs](https://github.com/GlobalDataverseCommunityConsortium/dataverse-language-packs). + +5\. Restart Payara + +```shell +service payara stop +service payara start +``` + +6\. Update metadata blocks + +These changes reflect incremental improvements made to the handling of core metadata fields. + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/scripts/api/data/metadatablocks/citation.tsv + +curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv ``` + +7\. Update Solr schema.xml file. See specific instructions below for those installations without custom metadata blocks (7a) and those with custom metadata blocks (7b). + +7a\. For installations without custom or experimental metadata blocks: + +Stop Solr (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.4/installation/prerequisites.html#solr-init-script)). + +```shell +service solr stop +``` + +Replace schema.xml + +```shell +wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/conf/solr/schema.xml +cp schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf +``` + +Start Solr. + +```shell +service solr start +``` + +7b\. For installations with custom or experimental metadata blocks: + +Stop Solr. + +```shell +service solr stop +``` + +There are two ways to regenerate the schema, either by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed)... + +```shell + wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.4.1/update-fields.sh + chmod +x update-fields.sh + curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml +``` + +...OR, alternatively, you edit your schema.xml by hand and add the following lines from the [6.4 tagged version](https://github.com/IQSS/dataverse/blob/v6.4/conf/solr/schema.xml) of schema.xml. + +``` + + + +``` + +Start Solr. + +```shell +service solr start +``` + +8\. Reindex Solr + +Below is the simple way to reindex Solr. If you have a large installation of Dataverse, you might want to reindex in place, as described in [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html). + +```shell +curl http://localhost:8080/api/admin/index +``` + +9\. Run reExportAll to update dataset metadata exports + +This step is necessary because of changes described above for the `oai_dc` export format. + +Below is the simple way to reexport all dataset metadata. For more advanced usage, please see [the guides](http://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api). + +```shell +curl http://localhost:8080/api/admin/metadata/reExportAll +``` + +### Additional Upgrade Steps + +10\. If there are broken thumbnails + +To restore any broken thumbnails caused by the bug described above, you can call the `http://localhost:8080/api/admin/clearThumbnailFailureFlag` API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the `http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID` to clear the flag for individual files. Calling the former, batch API is recommended. + +```shell +http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID +``` + +11\. PermaLinks with custom base-url + +If you currently use PermaLinks with a custom `base-url`: You must manually append `/citation?persistentId=` to the base URL to maintain functionality. + +If you use a PermaLinks without a configured `base-url`, no changes are required. From 79b4204e743e9fc842d3f23fd358c0df76abccd5 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 20 Sep 2024 13:17:34 -0400 Subject: [PATCH 09/44] Update 6.4-release-notes.md corrected the schema.xml instructions --- doc/release-notes/6.4-release-notes.md | 31 +++++--------------------- 1 file changed, 5 insertions(+), 26 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index f6013cbe2d6..8072741033c 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -374,9 +374,7 @@ wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/scripts/api/data/meta curl http://localhost:8080/api/admin/datasetfield/load -H "Content-type: text/tab-separated-values" -X POST --upload-file citation.tsv ``` -7\. Update Solr schema.xml file. See specific instructions below for those installations without custom metadata blocks (7a) and those with custom metadata blocks (7b). - -7a\. For installations without custom or experimental metadata blocks: +7\. Update Solr schema.xml file. Start with the standard v6.4 schema.xml, then, if your installation uses any custom or experimental metadata blocks, update it to include the extra fields (step 7a). Stop Solr (usually `service solr stop`, depending on Solr installation/OS, see the [Installation Guide](https://guides.dataverse.org/en/6.4/installation/prerequisites.html#solr-init-script)). @@ -391,41 +389,22 @@ wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/conf/solr/schema.xml cp schema.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf ``` -Start Solr. +Start Solr (but if you use any custom metadata blocks, perform the next step, 7a first). ```shell service solr start ``` -7b\. For installations with custom or experimental metadata blocks: - -Stop Solr. - -```shell -service solr stop -``` +7a\. For installations with custom or experimental metadata blocks: -There are two ways to regenerate the schema, either by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed)... +Before starting Solr, update the schema to include all the extra metadata fields that your installation uses. We do this by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the names of the directories, if different): ```shell wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.4.1/update-fields.sh chmod +x update-fields.sh curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml -``` - -...OR, alternatively, you edit your schema.xml by hand and add the following lines from the [6.4 tagged version](https://github.com/IQSS/dataverse/blob/v6.4/conf/solr/schema.xml) of schema.xml. -``` - - - -``` - -Start Solr. - -```shell -service solr start -``` +Now start Solr. 8\. Reindex Solr From b9f1c85ad0ae8666403367287790b96a50c5a563 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 20 Sep 2024 13:57:48 -0400 Subject: [PATCH 10/44] add blurb for tagged base images #10853 --- doc/release-notes/10478-version-base-image.md | 7 ------- doc/release-notes/6.4-release-notes.md | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) delete mode 100644 doc/release-notes/10478-version-base-image.md diff --git a/doc/release-notes/10478-version-base-image.md b/doc/release-notes/10478-version-base-image.md deleted file mode 100644 index 34f444a2122..00000000000 --- a/doc/release-notes/10478-version-base-image.md +++ /dev/null @@ -1,7 +0,0 @@ -### Adding versioned tags to Container Base Images - -With this release we introduce a detailed maintenance workflow for our container images. -As output of the GDCC Containerization Working Group, the community takes another step towards production ready containers available directly from the core project. - -The maintenance workflow regularly updates the Container Base Image, which contains the operating system, Java, Payara Application Server, as well as tools and libraries required by the Dataverse application. -Shipping these rolling releases as well as immutable revisions is the foundation for secure and reliable Dataverse Application Container images. diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 8072741033c..3c7c5fc4e8f 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -99,6 +99,12 @@ JSON Schema validation has been enhanced with checks for required and allowed ch Counter Processor 1.05 is now supported for use with Make Data Count. If you are running Counter Processor, you should reinstall/reconfigure it as described in the latest guides. Note that Counter Processor 1.05 requires Python 3, so you will need to follow the full Counter Processor install. Also note that if you configure the new version the same way, it will reprocess the days in the current month when it is first run. This is normal and will not affect the metrics in Dataverse. See also #10479. +### Version Tags for Container Base Images + +With this release we introduce a detailed maintenance workflow for our container images. As output of the [Containerization Working Group](https://ct.gdcc.io), the community takes another step towards production ready containers available directly from the core project. + +The maintenance workflow regularly updates the [Container Base Image](https://guides.dataverse.org/en/6.4/container/base-image.html), which contains the operating system, Java, Payara, and tools and libraries required by the Dataverse application. Shipping these rolling releases as well as immutable revisions is the foundation for secure and reliable [Dataverse Application Container](https://guides.dataverse.org/en/6.4/container/app-image.html) images. See also #10478 and #10827. + ## Bugs Fixed ### Update Current Version From 14efc334b2b323466387f24d6e860d4e3e0df288 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 23 Sep 2024 15:21:06 -0400 Subject: [PATCH 11/44] add more and better DataCite export #10853 --- .../10632-DataCiteXMLandRelationType.md | 41 ----------------- doc/release-notes/6.4-release-notes.md | 44 +++++++++++++++++-- 2 files changed, 40 insertions(+), 45 deletions(-) delete mode 100644 doc/release-notes/10632-DataCiteXMLandRelationType.md diff --git a/doc/release-notes/10632-DataCiteXMLandRelationType.md b/doc/release-notes/10632-DataCiteXMLandRelationType.md deleted file mode 100644 index 42c1cfb6eda..00000000000 --- a/doc/release-notes/10632-DataCiteXMLandRelationType.md +++ /dev/null @@ -1,41 +0,0 @@ -### Enhanced DataCite Metadata, Relation Type - -A new field has been added to the citation metadatablock to allow entry of the "Relation Type" between a "Related Publication" and a dataset. The Relation Type is currently limited to the most common 6 values recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. - -Dataverse now supports the DataCite v4.5 schema. Additional metadata, including metadata about Related Publications, and files in the dataset are now being sent to DataCite and improvements to how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata is represented have been made. The enhanced metadata will automatically be sent when datasets are created and published and is available in the DataCite XML export after publication. - -The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. For details see https://github.com/IQSS/dataverse/pull/10632 and https://github.com/IQSS/dataverse/pull/10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. - -Multiple backward incompatible changes and bug fixes have been made to API calls (3 of the four of which were not documented) related to updating PID target urls and metadata at the provider service: -- [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) -- [Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) -- [Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) -- [Update Metadata for all Published Datasets at the PID provider](https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) - -Upgrade instructions --------------------- - -The Solr schema has to be updated via the normal mechanism to add the new "relationType" field. - -The citation metadatablock has to be reinstalled using the standard instructions. - -With these two changes, the "Relation Type" fields will be available and creation/publication of datasets will result in the expanded XML being sent to DataCite. - -To update existing datasets (and files using DataCite DOIs): - -Exports can be updated by running `curl http://localhost:8080/api/admin/metadata/reExportAll` - -Entries at DataCite for published datasets can be updated by a superuser using an API call (newly documented): - -`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets/modifyRegistrationPIDMetadataAll` - -This will loop through all published datasets (and released files with PIDs). As long as the loop completes, the call will return a 200/OK response. Any PIDs for which the update fails can be found using - -`grep 'Failure for id' server.log` - -Failures may occur if PIDs were never registered, or if they were never made findable. Any such cases can be fixed manually in DataCite Fabrica or using the [Reserve a PID](https://guides.dataverse.org/en/latest/api/native-api.html#reserve-a-pid) API call and the newly documented `/api/datasets//modifyRegistration` call respectively. See https://guides.dataverse.org/en/latest/admin/dataverses-datasets.html#send-dataset-metadata-to-pid-provider. Please reach out with any questions. - -PIDs can also be updated by a superuser on a per-dataset basis using - -`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets//modifyRegistrationMetadata` - diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 3c7c5fc4e8f..fa50a00391b 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -13,6 +13,7 @@ Dataverse 6.4 fixes a couple important bugs described below and in [a post](http New features include: +- More and better metadata is being sent to DataCite, including "Relation Type" - All ISO 639-3 languages are now supported - There is now a button for "Unlink Dataset" - Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time @@ -25,6 +26,20 @@ All this and more can be found below. Read on! ## Features Added +### More and Better Data Sent to DataCite, Including "Relation Type" + +Within the "Related Publication" field, a new subfield has been added called "Relation Type" that allows for the most common [values](https://datacite-metadata-schema.readthedocs.io/en/4.5/appendices/appendix-1/relationType/) recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. + +Dataverse now supports the [DataCite v4.5 schema](http://schema.datacite.org/meta/kernel-4/). Additional metadata is now being sent to DataCite including metadata about related publications and files in the dataset. Improvement metadata is being sent including how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata is represented have been made. The enhanced metadata will automatically be sent to DataCite when datasets are created and published. Additionally, after publication, you can inspect what was sent by looking at the DataCite XML export. (Previously the export and what was sent wasn't 100% in sync.) + +The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. For details see #10632, #10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. + +Multiple backward incompatible changes and bug fixes have been made to API calls (three of the four of which were not documented) related to updating PID target urls and metadata at the provider service: +- [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) +- [Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) +- [Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) +- [Update Metadata for all Published Datasets at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider) + ### Full List of ISO 639-3 Languages Now Supported The controlled vocabulary values list for the metadata field "Language" in the citation block has now been extended to include roughly 7920 ISO 639-3 values. @@ -295,7 +310,8 @@ The following settings have been added: ## Backward Incompatible Changes -The oai_dc export format has changed. See the "Remap oai_dc" section above. +- The oai_dc export format has changed. See the "Remap oai_dc" section above. +- Several APIs related to DataCite have changed. See "More and Better Data Sent to DataCite" above. ## Complete List of Changes @@ -422,7 +438,7 @@ curl http://localhost:8080/api/admin/index 9\. Run reExportAll to update dataset metadata exports -This step is necessary because of changes described above for the `oai_dc` export format. +This step is necessary because of changes described above for the `Datacite` and `oai_dc` export formats. Below is the simple way to reexport all dataset metadata. For more advanced usage, please see [the guides](http://guides.dataverse.org/en/6.4/admin/metadataexport.html#batch-exports-through-the-api). @@ -430,9 +446,29 @@ Below is the simple way to reexport all dataset metadata. For more advanced usag curl http://localhost:8080/api/admin/metadata/reExportAll ``` +10\. Pushing updated metadata to DataCite + +(If you don't use DataCite, you can skip this.) + +Above you updated the citation metadata block and Solr with the new "relationType" field. With these two changes, the "Relation Type" fields will be available and creation/publication of datasets will result in the expanded XML being sent to DataCite. You've also already run "reExportAll" to update the `Datacite` metadata export format. + +Entries at DataCite for published datasets can be updated by a superuser using an API call (newly [documented](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-all-published-datasets-at-the-pid-provider)): + +`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets/modifyRegistrationPIDMetadataAll` + +This will loop through all published datasets (and released files with PIDs). As long as the loop completes, the call will return a 200/OK response. Any PIDs for which the update fails can be found using the following command: + +`grep 'Failure for id' server.log` + +Failures may occur if PIDs were never registered, or if they were never made findable. Any such cases can be fixed manually in DataCite Fabrica or using the [Reserve a PID](https://guides.dataverse.org/en/6.4/api/native-api.html#reserve-a-pid) API call and the newly documented `/api/datasets//modifyRegistration` call respectively. See https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#send-dataset-metadata-to-pid-provider. Please reach out with any questions. + +PIDs can also be updated by a superuser on a per-dataset basis using + +`curl -X POST -H 'X-Dataverse-key:' http://localhost:8080/api/datasets//modifyRegistrationMetadata` + ### Additional Upgrade Steps -10\. If there are broken thumbnails +11\. If there are broken thumbnails To restore any broken thumbnails caused by the bug described above, you can call the `http://localhost:8080/api/admin/clearThumbnailFailureFlag` API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the `http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID` to clear the flag for individual files. Calling the former, batch API is recommended. @@ -440,7 +476,7 @@ To restore any broken thumbnails caused by the bug described above, you can call http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID ``` -11\. PermaLinks with custom base-url +12\. PermaLinks with custom base-url If you currently use PermaLinks with a custom `base-url`: You must manually append `/citation?persistentId=` to the base URL to maintain functionality. From c3705487f1834a3ac7c8ddf53abf65c06d713fc5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 23 Sep 2024 15:25:26 -0400 Subject: [PATCH 12/44] add blur for listing feature flags #10853 --- doc/release-notes/6.4-release-notes.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index fa50a00391b..11fd7785a12 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -296,6 +296,10 @@ The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. +### Feature Flags Can Be Listed + +It now possible to list all feature flags and see if they are enabled or not. See #10732 and [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags). + ## Settings Added The following settings have been added: From 09608c1abb28eaab3bf13401b5e282f814a867d4 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 23 Sep 2024 16:27:09 -0400 Subject: [PATCH 13/44] add CVOC bug fix #10853 -6.4-release-notes --- doc/release-notes/10869-fix-npe-using-cvoc.md | 1 - doc/release-notes/6.4-release-notes.md | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 doc/release-notes/10869-fix-npe-using-cvoc.md diff --git a/doc/release-notes/10869-fix-npe-using-cvoc.md b/doc/release-notes/10869-fix-npe-using-cvoc.md deleted file mode 100644 index 53214d3789d..00000000000 --- a/doc/release-notes/10869-fix-npe-using-cvoc.md +++ /dev/null @@ -1 +0,0 @@ -This release fixes a bug in the external controlled vocabulary mechanism (introduced in v6.3) that could cause indexing to fail when a script is configured for one child field and no other child fields were managed. \ No newline at end of file diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 11fd7785a12..5ee2d4b4612 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -178,6 +178,10 @@ In order for these changes to be reflected in existing datasets, a [reexport all When detecting files types, Dataverse would previously detect a zip file as a shapefile if it contained [markers of a shapefile](https://guides.dataverse.org/en/6.4/developers/geospatial.html) in hidden directories. These hidden directories are now ignored when deciding if a zip file is a shapefile or not. See also #8945 and #10627. +### External Controlled Vocabulary + +This release fixes a bug (introduced in v6.3) in the external controlled vocabulary mechanism that could cause indexing to fail (with a NullPointerException) when a script is configured for one child field and no other child fields were managed. See also #10869 and #10870. + ### Valid JSON in Error Response When any `ApiBlockingFilter` policy applies to a request, the JSON in the body of the error response is now valid JSON. See also #10085. From 3be6ce44749881eb32b6e71aa4147383961900f8 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Sep 2024 09:50:03 -0400 Subject: [PATCH 14/44] update image_url #10853 --- ...831-standardize-image-url-of-search-api.md | 28 ------------------- doc/release-notes/6.4-release-notes.md | 6 ++-- 2 files changed, 3 insertions(+), 31 deletions(-) delete mode 100644 doc/release-notes/10831-standardize-image-url-of-search-api.md diff --git a/doc/release-notes/10831-standardize-image-url-of-search-api.md b/doc/release-notes/10831-standardize-image-url-of-search-api.md deleted file mode 100644 index 1910091455c..00000000000 --- a/doc/release-notes/10831-standardize-image-url-of-search-api.md +++ /dev/null @@ -1,28 +0,0 @@ -Search API (/api/search) response will now include new image_url format for the Datafile and Dataverse logo. -Note to release note writer: this supersedes the release note 10810-search-api-payload-extensions.md - -For Dataverse: - -- "image_url" (optional) - -```javascript -"items": [ - { - "name": "Darwin's Finches", - ... - "image_url":"/api/access/dvCardImage/{identifier}" -(etc, etc) -``` - -For DataFile: - -- "image_url" (optional) - -```javascript -"items": [ - { - "name": "test.txt", - ... - "image_url":"/api/access/datafile/{identifier}?imageThumb=true" -(etc, etc) -``` diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 5ee2d4b4612..a1d4eee042b 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -222,7 +222,7 @@ For collections (dataverses): "affiliation": "Dataverse.org", "parentDataverseName": "Root", "parentDataverseIdentifier": "root", - "image_url":"data:image/png;base64,iVBORw0..." + "image_url":"/api/access/dvCardImage/{identifier}" (etc, etc) ``` @@ -247,10 +247,10 @@ For files: ```javascript "items": [ { - "name": "test.txt", + "name": "test.png", ... "releaseOrCreateDate": "2016-05-10T12:53:39Z", - "image_url":"data:image/png;base64,iVBORw0..." + "image_url":"/api/access/datafile/42?imageThumb=true" (etc, etc) ``` From 110fbe10dd57d6c881a31ceb602c78470aed2ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Kir=C3=A1ly?= Date: Tue, 20 Jun 2023 17:35:58 +0200 Subject: [PATCH 15/44] #9650 improve the linked dataverse listing API --- .../java/edu/harvard/iq/dataverse/api/Datasets.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index f26b6d67edf..1700c4fbcf5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2076,10 +2076,16 @@ public Response getLinks(@Context ContainerRequestContext crc, @PathParam("id") List dvsThatLinkToThisDatasetId = dataverseSvc.findDataversesThatLinkToThisDatasetId(datasetId); JsonArrayBuilder dataversesThatLinkToThisDatasetIdBuilder = Json.createArrayBuilder(); for (Dataverse dataverse : dvsThatLinkToThisDatasetId) { - dataversesThatLinkToThisDatasetIdBuilder.add(dataverse.getAlias() + " (id " + dataverse.getId() + ")"); + JsonObjectBuilder entry = Json.createObjectBuilder(); + entry.add("id", dataverse.getId()); + entry.add("alias", dataverse.getAlias()); + entry.add("displayName", dataverse.getDisplayName()); + dataversesThatLinkToThisDatasetIdBuilder.add(entry); } JsonObjectBuilder response = Json.createObjectBuilder(); - response.add("dataverses that link to dataset id " + datasetId, dataversesThatLinkToThisDatasetIdBuilder); + response.add("id", datasetId); + response.add("identifier", dataset.getIdentifier()); + response.add("linked-dataverses", dataversesThatLinkToThisDatasetIdBuilder); return ok(response); } catch (WrappedResponse wr) { return wr.getResponse(); From 2c2cd86f7a1d0cf994cded37ffbf71e66bb824cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Kir=C3=A1ly?= Date: Tue, 20 Jun 2023 18:29:18 +0200 Subject: [PATCH 16/44] #9650 backwards compatibility, release note --- .../9650-5-improve-list-linked-dataverses-API.md | 5 +++++ src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 doc/release-notes/9650-5-improve-list-linked-dataverses-API.md diff --git a/doc/release-notes/9650-5-improve-list-linked-dataverses-API.md b/doc/release-notes/9650-5-improve-list-linked-dataverses-API.md new file mode 100644 index 00000000000..8c79955891b --- /dev/null +++ b/doc/release-notes/9650-5-improve-list-linked-dataverses-API.md @@ -0,0 +1,5 @@ +The following API have been added: + +/api/datasets/{datasetId}/links + +It lists the linked dataverses to a dataset. It can be executed only by administrators. \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 1700c4fbcf5..24b20b8b882 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2085,7 +2085,7 @@ public Response getLinks(@Context ContainerRequestContext crc, @PathParam("id") JsonObjectBuilder response = Json.createObjectBuilder(); response.add("id", datasetId); response.add("identifier", dataset.getIdentifier()); - response.add("linked-dataverses", dataversesThatLinkToThisDatasetIdBuilder); + response.add("dataverses that link to dataset id " + datasetId, dataversesThatLinkToThisDatasetIdBuilder); return ok(response); } catch (WrappedResponse wr) { return wr.getResponse(); From 6228d3f030a331487e9237958626f171cb52c623 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 12 Sep 2024 12:59:33 -0400 Subject: [PATCH 17/44] adding tests, doc changes, and formatting fix --- .../source/admin/dataverses-datasets.rst | 12 ++++++-- .../harvard/iq/dataverse/api/Datasets.java | 12 ++++---- .../harvard/iq/dataverse/api/DatasetsIT.java | 28 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/MoveIT.java | 7 ++--- 4 files changed, 46 insertions(+), 13 deletions(-) diff --git a/doc/sphinx-guides/source/admin/dataverses-datasets.rst b/doc/sphinx-guides/source/admin/dataverses-datasets.rst index b4d365c4fd4..7c03a6f80c0 100644 --- a/doc/sphinx-guides/source/admin/dataverses-datasets.rst +++ b/doc/sphinx-guides/source/admin/dataverses-datasets.rst @@ -129,15 +129,21 @@ Lists the link(s) created between a dataset and a Dataverse collection (see the curl -H "X-Dataverse-key: $API_TOKEN" http://$SERVER/api/datasets/$linked-dataset-id/links -It returns a list in the following format: +It returns a list in the following format (new format as of v6.4): .. code-block:: json { "status": "OK", "data": { - "dataverses that link to dataset id 56782": [ - "crc990 (id 18802)" + "id": 5, + "identifier": "FK2/OTCWMM", + "linked-dataverses": [ + { + "id": 2, + "alias": "dataverse1", + "displayName": "Lab experiments 2023 June" + } ] } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 24b20b8b882..e0d3bf6126e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2076,16 +2076,16 @@ public Response getLinks(@Context ContainerRequestContext crc, @PathParam("id") List dvsThatLinkToThisDatasetId = dataverseSvc.findDataversesThatLinkToThisDatasetId(datasetId); JsonArrayBuilder dataversesThatLinkToThisDatasetIdBuilder = Json.createArrayBuilder(); for (Dataverse dataverse : dvsThatLinkToThisDatasetId) { - JsonObjectBuilder entry = Json.createObjectBuilder(); - entry.add("id", dataverse.getId()); - entry.add("alias", dataverse.getAlias()); - entry.add("displayName", dataverse.getDisplayName()); - dataversesThatLinkToThisDatasetIdBuilder.add(entry); + JsonObjectBuilder datasetBuilder = Json.createObjectBuilder(); + datasetBuilder.add("id", dataverse.getId()); + datasetBuilder.add("alias", dataverse.getAlias()); + datasetBuilder.add("displayName", dataverse.getDisplayName()); + dataversesThatLinkToThisDatasetIdBuilder.add(datasetBuilder.build()); } JsonObjectBuilder response = Json.createObjectBuilder(); response.add("id", datasetId); response.add("identifier", dataset.getIdentifier()); - response.add("dataverses that link to dataset id " + datasetId, dataversesThatLinkToThisDatasetIdBuilder); + response.add("linked-dataverses", dataversesThatLinkToThisDatasetIdBuilder); return ok(response); } catch (WrappedResponse wr) { return wr.getResponse(); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index f52aa4fe9bd..b7bdaeebbfc 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2963,6 +2963,34 @@ public void testLinkingDatasets() { linkDataset.then().assertThat() .statusCode(OK.getStatusCode()); + // Link another to test the list of linked datasets + Response createDataverse3 = UtilIT.createRandomDataverse(apiToken); + createDataverse3.prettyPrint(); + createDataverse3.then().assertThat() + .statusCode(CREATED.getStatusCode()); + String dataverse3Alias = UtilIT.getAliasFromResponse(createDataverse3); + Integer dataverse3Id = UtilIT.getDatasetIdFromResponse(createDataverse3); + linkDataset = UtilIT.linkDataset(datasetPid, dataverse3Alias, superuserApiToken); + linkDataset.prettyPrint(); + linkDataset.then().assertThat() + .statusCode(OK.getStatusCode()); + // get the list in Json format + Response linkDatasetsResponse = UtilIT.getDatasetLinks(datasetPid, superuserApiToken); + linkDatasetsResponse.prettyPrint(); + linkDatasetsResponse.then().assertThat() + .statusCode(OK.getStatusCode()); + JsonObject linkDatasets = Json.createReader(new StringReader(linkDatasetsResponse.asString())).readObject(); + JsonArray lst = linkDatasets.getJsonObject("data").getJsonArray("linked-dataverses"); + List ids = List.of(dataverse2Id, dataverse3Id); + List uniqueids = new ArrayList<>(); + assertEquals(ids.size(), lst.size()); + for (int i = 0; i < lst.size(); i++) { + int id = lst.getJsonObject(i).getInt("id"); + assertTrue(ids.contains(id)); + assertFalse(uniqueids.contains(id)); + uniqueids.add(id); + } + //Experimental code for trying to trick test into thinking the dataset has been harvested /* createDataset = UtilIT.createRandomDatasetViaNativeApi(dataverse1Alias, apiToken); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java index f7135ce7f3b..8951b0bd42e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MoveIT.java @@ -278,8 +278,8 @@ public void testMoveLinkedDataset() { .body("message", equalTo("Use the query parameter forceMove=true to complete the move. This dataset is linked to the new host dataverse or one of its parents. This move would remove the link to this dataset. ")); JsonObject linksBeforeData = Json.createReader(new StringReader(getLinksBefore.asString())).readObject(); - assertEquals("OK", linksBeforeData.getString("status")); - assertEquals(dataverse2Alias + " (id " + dataverse2Id + ")", linksBeforeData.getJsonObject("data").getJsonArray("dataverses that link to dataset id " + datasetId).getString(0)); + assertEquals(datasetId, linksBeforeData.getJsonObject("data").getInt("id")); + assertEquals(dataverse2Id, linksBeforeData.getJsonObject("data").getJsonArray("linked-dataverses").get(0).asJsonObject().getInt("id")); boolean forceMove = true; Response forceMoveLinkedDataset = UtilIT.moveDataset(datasetId.toString(), dataverse2Alias, forceMove, superuserApiToken); @@ -308,8 +308,7 @@ public void testMoveLinkedDataset() { JsonObject linksAfterData = Json.createReader(new StringReader(getLinksAfter.asString())).readObject(); assertEquals("OK", linksAfterData.getString("status")); - assertEquals(0, linksAfterData.getJsonObject("data").getJsonArray("dataverses that link to dataset id " + datasetId).size()); - + assertEquals(0, linksAfterData.getJsonObject("data").getJsonArray("linked-dataverses").size()); } @Test From 54fad55ed773dab96f023ee95dc980294289cec5 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Thu, 12 Sep 2024 14:42:30 -0400 Subject: [PATCH 18/44] update doc --- doc/sphinx-guides/source/api/changelog.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/changelog.rst b/doc/sphinx-guides/source/api/changelog.rst index c5890fd9996..92cd4fc941b 100644 --- a/doc/sphinx-guides/source/api/changelog.rst +++ b/doc/sphinx-guides/source/api/changelog.rst @@ -12,7 +12,7 @@ v6.4 - **/api/datasets/$dataset-id/modifyRegistration**: Changed from GET to POST - **/api/datasets/modifyRegistrationPIDMetadataAll**: Changed from GET to POST - +- **/api/datasets/{identifier}/links**: The GET endpoint returns a list of Dataverses linked to the given Dataset. The format of the response has changes for v6.4 making it backward incompatible. v6.3 ---- From def801ef45cb8a6ff639bb096410f8710af6e785 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Sep 2024 14:54:23 -0400 Subject: [PATCH 19/44] add globus async #10853 --- doc/release-notes/10623-globus-improvements.md | 1 - doc/release-notes/6.4-release-notes.md | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 doc/release-notes/10623-globus-improvements.md diff --git a/doc/release-notes/10623-globus-improvements.md b/doc/release-notes/10623-globus-improvements.md deleted file mode 100644 index 9eb529bc4f7..00000000000 --- a/doc/release-notes/10623-globus-improvements.md +++ /dev/null @@ -1 +0,0 @@ -A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://dataverse-guide--10781.org.readthedocs.build/en/10781/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://dataverse-guide--10781.org.readthedocs.build/en/10781/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index a1d4eee042b..b0dcdd1fcf7 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -98,6 +98,10 @@ It is now possible to turn off the feature that automatically selects of one of The configuration setting `dataverse.pid.*.permalink.base-url`, which is used for PermaLinks, has been updated to support greater flexibility. Previously, the string `/citation?persistentId=` was automatically appended to the configured base URL. With this update, the base URL will now be used exactly as configured, without any automatic additions. See also #10775. +### Globus Async Framework + +A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://guides.dataverse.org/en/6.4/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. + ### Development on Windows New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See [the guides](https://guides.dataverse.org/en/6.4/developers/windows.html), #10606, and #10608. From c2865280cac9ae67a8f01e8b1c6308dacbb0f924 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 25 Sep 2024 16:27:47 -0400 Subject: [PATCH 20/44] add cvoc update #10853 --- doc/release-notes/10711-CVoc Updates.md | 1 - doc/release-notes/6.4-release-notes.md | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) delete mode 100644 doc/release-notes/10711-CVoc Updates.md diff --git a/doc/release-notes/10711-CVoc Updates.md b/doc/release-notes/10711-CVoc Updates.md deleted file mode 100644 index f747bedb049..00000000000 --- a/doc/release-notes/10711-CVoc Updates.md +++ /dev/null @@ -1 +0,0 @@ -Changes in Dataverse and updates to the ORCID and ROR external vocabulary scripts support deploying these for the citation block author field (and others). diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index b0dcdd1fcf7..7115d26a505 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -102,6 +102,10 @@ The configuration setting `dataverse.pid.*.permalink.base-url`, which is used fo A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://guides.dataverse.org/en/6.4/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. +### CVoc (Controlled Vocabulary): Allow ORCID and ROR to be Used Together in Author Field + +Changes in Dataverse and updates to the ORCID and ROR external vocabulary scripts support deploying these for the citation block author field (and others). See also #10711, #10712, and . + ### Development on Windows New instructions have been added for developers on Windows trying to run a Dataverse development environment using Windows Subsystem for Linux (WSL). See [the guides](https://guides.dataverse.org/en/6.4/developers/windows.html), #10606, and #10608. From 62dd3486b42fc41eec9760cfcea917e4398ccefb Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 10:08:50 -0400 Subject: [PATCH 21/44] add new globus settings under settings section #10853 --- doc/release-notes/6.4-release-notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 7115d26a505..6bdc2625c46 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -317,6 +317,8 @@ It now possible to list all feature flags and see if they are enabled or not. Se The following settings have been added: - dataverse.feature.disable-dataset-thumbnail-autoselect +- dataverse.feature.globus-use-experimental-async-framework +- dataverse.files.globus-monitoring-server - dataverse.pid.*.crossref.url - dataverse.pid.*.crossref.rest-api-url - dataverse.pid.*.crossref.username From a01a0cbe989085b7dfa881ac6f5c69bea2a1be56 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 10:44:50 -0400 Subject: [PATCH 22/44] various tweaks #10853 --- doc/release-notes/6.4-release-notes.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 6bdc2625c46..a632e5e3d19 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -30,11 +30,11 @@ All this and more can be found below. Read on! Within the "Related Publication" field, a new subfield has been added called "Relation Type" that allows for the most common [values](https://datacite-metadata-schema.readthedocs.io/en/4.5/appendices/appendix-1/relationType/) recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. -Dataverse now supports the [DataCite v4.5 schema](http://schema.datacite.org/meta/kernel-4/). Additional metadata is now being sent to DataCite including metadata about related publications and files in the dataset. Improvement metadata is being sent including how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata is represented have been made. The enhanced metadata will automatically be sent to DataCite when datasets are created and published. Additionally, after publication, you can inspect what was sent by looking at the DataCite XML export. (Previously the export and what was sent wasn't 100% in sync.) +Dataverse now supports the [DataCite v4.5 schema](http://schema.datacite.org/meta/kernel-4/). Additional metadata is now being sent to DataCite including metadata about related publications and files in the dataset. Improved metadata is being sent including how PIDs (ORCID, ROR, DOIs, etc.), license/terms, geospatial, and other metadata are represented. The enhanced metadata will automatically be sent to DataCite when datasets are created and published. Additionally, after publication, you can inspect what was sent by looking at the DataCite XML export. The additions are in rough alignment with the OpenAIRE XML export, but there are some minor differences in addition to the Relation Type addition, including an update to the DataCite 4.5 schema. For details see #10632, #10615 and the [design document](https://docs.google.com/document/d/1JzDo9UOIy9dVvaHvtIbOI8tFU6bWdfDfuQvWWpC0tkA/edit?usp=sharing) referenced there. -Multiple backward incompatible changes and bug fixes have been made to API calls (three of the four of which were not documented) related to updating PID target urls and metadata at the provider service: +Multiple backward incompatible changes and bug fixes have been made to API calls (three of four of which were not documented) related to updating PID target URLs and metadata at the provider service: - [Update Target URL for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-a-published-dataset-at-the-pid-provider) - [Update Target URL for all Published Datasets at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-target-url-for-all-published-datasets-at-the-pid-provider) - [Update Metadata for a Published Dataset at the PID provider](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#update-metadata-for-a-published-dataset-at-the-pid-provider) @@ -50,7 +50,7 @@ ISO 639-3 codes were downloaded from [sil.org](https://iso639-3.sil.org/code_tab ### Unlink Dataset Button -New "Unlink Dataset" button has been added to the dataset page to allow a user to unlink a dataset from a collection. The user must possess the same permissions needed to unlink the dataset as they would to link the Dataset. Additionally, the [existing API](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#unlink-a-dataset) for unlinking datasets has been updated to no longer require superuser access as the "Publish Dataset" permission is now enough. See also #10583 and #10689. +A new "Unlink Dataset" button has been added to the dataset page to allow a user to unlink a dataset from a collection. To unlink a dataset the user must have permission to link the dataset. Additionally, the [existing API](https://guides.dataverse.org/en/6.4/admin/dataverses-datasets.html#unlink-a-dataset) for unlinking datasets has been updated to no longer require superuser access as the "Publish Dataset" permission is now enough. See also #10583 and #10689. ### Pre-Publish File DOI Reservation @@ -58,7 +58,7 @@ Dataverse installations using DataCite as a persistent identifier (PID) provider ### Initial Support for Dataset Types -Out of the box, all datasets now have the type "dataset" but superusers can add additional types. At this time the type of a dataset can only be set at creation time via API. The types "dataset", "software", and "workflow" (just those three, for now) will be sent to DataCite when the dataset is published. +Out of the box, all datasets now have the type "dataset" but superusers can add additional types. At this time the type of a dataset can only be set at creation time via API. The types "dataset", "software", and "workflow" (just those three, for now) will be sent to DataCite (as `resourceTypeGeneral`) when the dataset is published. For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-management.html#dataset-types), #10517 and #10694. Please note that this feature is highly experimental and is expected to [evolve](https://github.com/IQSS/dataverse-pm/issues/307). @@ -102,7 +102,7 @@ The configuration setting `dataverse.pid.*.permalink.base-url`, which is used fo A new alternative implementation of Globus polling during upload data transfers has been added in this release. This experimental framework does not rely on the instance staying up continuously for the duration of the transfer and saves the state information about Globus upload requests in the database. See `globus-use-experimental-async-framework` under [Feature Flags](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) and [dataverse.files.globus-monitoring-server](https://guides.dataverse.org/en/6.4/installation/config.html#dataverse-files-globus-monitoring-server) in the Installation Guide. See also #10623 and #10781. -### CVoc (Controlled Vocabulary): Allow ORCID and ROR to be Used Together in Author Field +### CVoc (Controlled Vocabulary): Allow ORCID and ROR to Be Used Together in Author Field Changes in Dataverse and updates to the ORCID and ROR external vocabulary scripts support deploying these for the citation block author field (and others). See also #10711, #10712, and . @@ -159,7 +159,7 @@ See also #10819, #10820, and [the post](https://groups.google.com/g/dataverse-co ### No License, No Terms of Use -When datasets have neither a license nor custom terms of use, the dataset page will indicate this. Also, these datasets will no longer be indexed as having custom terms. See also #8796, #10513, and #10614. +When datasets have neither a license nor custom terms of use, the dataset page will now indicate this. Also, these datasets will no longer be indexed as having custom terms. See also #8796, #10513, and #10614. ### CC0 License Bug Fix @@ -264,7 +264,7 @@ For files: These examples are also shown in the [Search API](https://guides.dataverse.org/en/6.4/api/search.html) section of the API Guide. -The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn't returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF. Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. +The image_url field was already part of the SolrSearchResult JSON (and incorrectly appeared in Search API documentation), but it wasn't returned by the API because it was appended only after the Solr query was executed in the SearchIncludeFragment of JSF (the old/current UI framework). Now, the field is set in SearchServiceBean, ensuring it is always returned by the API when an image is available. The Solr schema.xml file has been updated to include a new field called "dvParentAlias" for supporting the new response field "parentDataverseIdentifier". See upgrade instructions below. @@ -298,7 +298,7 @@ A new optional query parameter "returnDetails" added to `/api/dataverses/{identi ### User Permissions on Collections -A new endpoint at `/api/dataverses/{identifier}/userPermissions` for obtaining the user permissions on a collection (dataverse) has been added. See also #10749 and #10751. +A new endpoint at `/api/dataverses/{identifier}/userPermissions` for obtaining the user permissions on a collection (dataverse) has been added. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#get-user-permissions-on-a-dataverse), #10749 and #10751. ### addDataverse Extended @@ -310,7 +310,7 @@ The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not ### Feature Flags Can Be Listed -It now possible to list all feature flags and see if they are enabled or not. See #10732 and [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags). +It now possible to list all feature flags and see if they are enabled or not. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags) and #10732. ## Settings Added From 0177062665dc22c25a479caced80bfab7ef79fb6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 11:00:20 -0400 Subject: [PATCH 23/44] typo #10853 --- doc/release-notes/6.4-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index a632e5e3d19..7ab2aafcd95 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -310,7 +310,7 @@ The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not ### Feature Flags Can Be Listed -It now possible to list all feature flags and see if they are enabled or not. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags) and #10732. +It is now possible to list all feature flags and see if they are enabled or not. See also [the guides](https://guides.dataverse.org/en/6.4/api/native-api.html#list-all-feature-flags) and #10732. ## Settings Added From 7b0c5115679de1b71e9a0eb573eb3a2db8f6841a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 14:06:11 -0400 Subject: [PATCH 24/44] put features before bug fixes #10853 --- doc/release-notes/6.4-release-notes.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 7ab2aafcd95..5ed3341998f 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -6,12 +6,7 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ## Release Highlights -Dataverse 6.4 fixes a couple important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: - -- "Update Current Version" can cause metadata loss -- Publishing breaks designated dataset thumbnail, messes up collection page - -New features include: +Dataverse 6.4 includes many new features, including: - More and better metadata is being sent to DataCite, including "Relation Type" - All ISO 639-3 languages are now supported @@ -22,7 +17,12 @@ New features include: - RO-Crate support - A new Rust library -All this and more can be found below. Read on! +This release also fixes a couple important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: + +- "Update Current Version" can cause metadata loss +- Publishing breaks designated dataset thumbnail, messes up collection page + +Additional details on the above as well as many more features and bug fixes included in the release are described below. Read on! ## Features Added From e54e066ad4e4275fb1797753403533f3f04bd67e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 14:15:55 -0400 Subject: [PATCH 25/44] reword --- doc/release-notes/6.4-release-notes.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 5ed3341998f..c7c8856843e 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -6,7 +6,7 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than ## Release Highlights -Dataverse 6.4 includes many new features, including: +New features in Dataverse 6.4: - More and better metadata is being sent to DataCite, including "Relation Type" - All ISO 639-3 languages are now supported @@ -15,9 +15,13 @@ Dataverse 6.4 includes many new features, including: - Datasets can now have types such as "software" or "workflow" - Croissant support - RO-Crate support -- A new Rust library +- and more! Please see below. -This release also fixes a couple important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: +New client library: + +- Rust + +This release also fixes two important bugs described below and in [a post](https://groups.google.com/g/dataverse-community/c/evn5C-pyrS8/m/JrH9vp47DwAJ) on the mailing list: - "Update Current Version" can cause metadata loss - Publishing breaks designated dataset thumbnail, messes up collection page From c6d626dea0e93f110c35ac2016a36922f534dfc1 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 14:20:06 -0400 Subject: [PATCH 26/44] datacite title --- doc/release-notes/6.4-release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index c7c8856843e..60c9ab95adf 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -8,7 +8,7 @@ This release brings new features, enhancements, and bug fixes to Dataverse. Than New features in Dataverse 6.4: -- More and better metadata is being sent to DataCite, including "Relation Type" +- Enhanced DataCite Metadata, including "Relation Type" - All ISO 639-3 languages are now supported - There is now a button for "Unlink Dataset" - Users will have DOIs/PIDs reserved for their files as part of file upload instead of at publication time @@ -30,7 +30,7 @@ Additional details on the above as well as many more features and bug fixes incl ## Features Added -### More and Better Data Sent to DataCite, Including "Relation Type" +### Enhanced DataCite Metadata, Including "Relation Type" Within the "Related Publication" field, a new subfield has been added called "Relation Type" that allows for the most common [values](https://datacite-metadata-schema.readthedocs.io/en/4.5/appendices/appendix-1/relationType/) recommended by DataCite: isCitedBy, Cites, IsSupplementTo, IsSupplementedBy, IsReferencedBy, and References. For existing datasets where no "Relation Type" has been specified, "IsSupplementTo" is assumed. From 1654313d28596478a49e38c4f2640f4a26440bc5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Thu, 26 Sep 2024 14:27:45 -0400 Subject: [PATCH 27/44] croissant update #10853 --- doc/release-notes/6.4-release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 60c9ab95adf..648e628864d 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -70,10 +70,10 @@ For details see [the guides](https://guides.dataverse.org/en/6.4/user/dataset-ma A new metadata export format called [Croissant](https://github.com/mlcommons/croissant) is now available as an external metadata exporter. It is oriented toward making datasets consumable by machine learning. -When enabled, Croissant replaces the Schema.org JSON-LD format in the `` of dataset landing pages. For details, see the [Schema.org JSON-LD/Croissant Metadata](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) under the discoverability section of the Admin Guide. This change was made in Dataverse 6.3 in #10382. - For more about the Croissant exporter, including installation instructions, see . See also #10341, #10533, and [discussion](https://groups.google.com/g/dataverse-community/c/JI8HPgGarr8/m/DqEIkiwlAgAJ) on the mailing list. +Please note: the Croissant exporter works best with Dataverse 6.2 and higher (where it updates the content of `` as [described](https://guides.dataverse.org/en/6.4/admin/discoverability.html#schema-org-head) in the guides) but can be used with 6.0 and higher (to get the export functionality). + ### RO-Crate Support (Metadata Export) Dataverse now supports [RO-Crate](https://www.researchobject.org/ro-crate/) as a metadata export format. This functionality is not available out of the box, but you can enable one or more RO-Crate exporters from the [list of external exporters](https://guides.dataverse.org/en/6.4/installation/advanced.html#inventory-of-external-exporters). See also #10744 and #10796. From 1b2a52b5ed158686719afaee6c351535d33e4073 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Sep 2024 09:43:06 -0400 Subject: [PATCH 28/44] bump version to 6.4 #10852 (#10871) --- doc/sphinx-guides/source/conf.py | 4 ++-- doc/sphinx-guides/source/versions.rst | 3 ++- modules/dataverse-parent/pom.xml | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/doc/sphinx-guides/source/conf.py b/doc/sphinx-guides/source/conf.py index c719fb05e3c..7ee355302d8 100755 --- a/doc/sphinx-guides/source/conf.py +++ b/doc/sphinx-guides/source/conf.py @@ -68,9 +68,9 @@ # built documents. # # The short X.Y version. -version = '6.3' +version = '6.4' # The full version, including alpha/beta/rc tags. -release = '6.3' +release = '6.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/doc/sphinx-guides/source/versions.rst b/doc/sphinx-guides/source/versions.rst index 952eba72616..800bdc6e0f9 100755 --- a/doc/sphinx-guides/source/versions.rst +++ b/doc/sphinx-guides/source/versions.rst @@ -7,7 +7,8 @@ Dataverse Software Documentation Versions This list provides a way to refer to the documentation for previous and future versions of the Dataverse Software. In order to learn more about the updates delivered from one version to another, visit the `Releases `__ page in our GitHub repo. - pre-release `HTML (not final!) `__ and `PDF (experimental!) `__ built from the :doc:`develop ` branch :doc:`(how to contribute!) ` -- 6.3 +- 6.4 +- `6.3 `__ - `6.2 `__ - `6.1 `__ - `6.0 `__ diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 6bea02569ec..76f0ddbc033 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -131,7 +131,7 @@ - 6.3 + 6.4 17 UTF-8 @@ -446,8 +446,8 @@ Once the release has been made (tag created), change this back to "${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}" (These properties are provided by the build-helper plugin below.) --> - ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} - + + ${revision} From 068607793b70d6fdd0b0ee1b1a3d2a5bfc2c2574 Mon Sep 17 00:00:00 2001 From: Guillermo Portas Date: Fri, 27 Sep 2024 16:05:32 +0200 Subject: [PATCH 29/44] displayOnCreate set to true for depositor and dateOfDeposit in Citation metadata block (#10884) * Changed: displayOnCreate set to true for depositor and dateOfDeposit in citation.tsv * Changed: MetadataBlocksIT test assertion for new total number of displayOnCreate fields * Added: release notes for #10850 * Added: minor tweak to release notes --- ...50-citation-tsv-displayoncreate-depositor-dateofdeposit.md | 1 + scripts/api/data/metadatablocks/citation.tsv | 4 ++-- .../java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md diff --git a/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md b/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md new file mode 100644 index 00000000000..2e404fec75b --- /dev/null +++ b/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md @@ -0,0 +1 @@ +The fields depositor and dateOfDeposit in the citation.tsv metadata block file have been updated to have the property displayOnCreate set to TRUE. \ No newline at end of file diff --git a/scripts/api/data/metadatablocks/citation.tsv b/scripts/api/data/metadatablocks/citation.tsv index a9ea2b9ca0e..abc09465603 100644 --- a/scripts/api/data/metadatablocks/citation.tsv +++ b/scripts/api/data/metadatablocks/citation.tsv @@ -59,8 +59,8 @@ distributorURL URL The URL of the distributor's webpage https:// url 55 #VALUE FALSE FALSE FALSE FALSE FALSE FALSE distributor citation distributorLogoURL Logo URL The URL of the distributor's logo image, used to show the image on the Dataset's page https:// url 56
FALSE FALSE FALSE FALSE FALSE FALSE distributor citation distributionDate Distribution Date The date when the Dataset was made available for distribution/presentation YYYY-MM-DD date 57 TRUE FALSE FALSE TRUE FALSE FALSE citation - depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 58 FALSE FALSE FALSE FALSE FALSE FALSE citation - dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 59 FALSE FALSE FALSE TRUE FALSE FALSE citation http://purl.org/dc/terms/dateSubmitted + depositor Depositor The entity, such as a person or organization, that deposited the Dataset in the repository 1) FamilyName, GivenName or 2) Organization text 58 FALSE FALSE FALSE FALSE TRUE FALSE citation + dateOfDeposit Deposit Date The date when the Dataset was deposited into the repository YYYY-MM-DD date 59 FALSE FALSE FALSE TRUE TRUE FALSE citation http://purl.org/dc/terms/dateSubmitted timePeriodCovered Time Period The time period that the data refer to. Also known as span. This is the time period covered by the data, not the dates of coding, collecting data, or making documents machine-readable none 60 ; FALSE FALSE TRUE FALSE FALSE FALSE citation https://schema.org/temporalCoverage timePeriodCoveredStart Start Date The start date of the time period that the data refer to YYYY-MM-DD date 61 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation timePeriodCoveredEnd End Date The end date of the time period that the data refer to YYYY-MM-DD date 62 #NAME: #VALUE TRUE FALSE FALSE TRUE FALSE FALSE timePeriodCovered citation diff --git a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java index 0153d8dc893..6e7061961f0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/MetadataBlocksIT.java @@ -51,7 +51,7 @@ void testListMetadataBlocks() { // onlyDisplayedOnCreate=true and returnDatasetFieldTypes=true listMetadataBlocksResponse = UtilIT.listMetadataBlocks(true, true); - expectedNumberOfMetadataFields = 26; + expectedNumberOfMetadataFields = 28; listMetadataBlocksResponse.then().assertThat() .statusCode(OK.getStatusCode()) .body("data[0].fields", not(equalTo(null))) From 39b6aa28c0befcf316a84117e2ef9af5a250f028 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Fri, 27 Sep 2024 10:22:19 -0400 Subject: [PATCH 30/44] #10853 fix typo version number --- doc/release-notes/6.4-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 648e628864d..7d64ce898c7 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -355,7 +355,7 @@ You are also very welcome to join the [Global Dataverse Community Consortium](ht Upgrading requires a maintenance window and downtime. Please plan accordingly, create backups of your database, etc. -These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.2. +These instructions assume that you've already upgraded through all the 5.x releases and are now running Dataverse 6.3. 0\. These instructions assume that you are upgrading from the immediate previous version. If you are running an earlier version, the only supported way to upgrade is to progress through the upgrades to all the releases in between before attempting the upgrade to this version. From 56118add8d464c1574b752f42486e255cc75e7c7 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:34:01 -0400 Subject: [PATCH 31/44] Add release note change for fields depositor and dateOfDeposit in the citation.tsv --- doc/release-notes/6.4-release-notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 7d64ce898c7..693ec4521e1 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -311,6 +311,7 @@ The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended ### Metadata Blocks and Display on Create The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. +The fields depositor and dateOfDeposit in the citation.tsv metadata block file have been updated to have the property displayOnCreate set to TRUE. ### Feature Flags Can Be Listed From 423d4f3707c27cee16ee96fe19892836a619dccd Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 27 Sep 2024 10:52:20 -0400 Subject: [PATCH 32/44] remove old release note --- ...10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md diff --git a/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md b/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md deleted file mode 100644 index 2e404fec75b..00000000000 --- a/doc/release-notes/10850-citation-tsv-displayoncreate-depositor-dateofdeposit.md +++ /dev/null @@ -1 +0,0 @@ -The fields depositor and dateOfDeposit in the citation.tsv metadata block file have been updated to have the property displayOnCreate set to TRUE. \ No newline at end of file From b6abba9e056e5a3ef4ba3f3f7d36be64bc047774 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 10:54:36 -0400 Subject: [PATCH 33/44] formatting fix fixed formatting of the shell block in the upgrade instruction --- doc/release-notes/6.4-release-notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 693ec4521e1..a3027ece5d8 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -448,6 +448,7 @@ Before starting Solr, update the schema to include all the extra metadata fields wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.4.1/update-fields.sh chmod +x update-fields.sh curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml +``` Now start Solr. From 8e0496d2d16d43670b9a55aca6c18f8efe352159 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 27 Sep 2024 11:07:13 -0400 Subject: [PATCH 34/44] tweak depositor and dateOfDeposit #10853 --- doc/release-notes/6.4-release-notes.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index a3027ece5d8..4268f6fb56e 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -311,7 +311,8 @@ The addDataverse (`/api/dataverses/{identifier}`) API endpoint has been extended ### Metadata Blocks and Display on Create The `/api/dataverses/{identifier}/metadatablocks` endpoint has been fixed to not return fields marked as displayOnCreate=true if there is an input level with include=false, when query parameters returnDatasetFieldTypes=true and onlyDisplayedOnCreate=true are set. See also #10741 and #10767. -The fields depositor and dateOfDeposit in the citation.tsv metadata block file have been updated to have the property displayOnCreate set to TRUE. + +The fields "depositor" and "dateOfDeposit" in the citation.tsv metadata block file have been updated to have the property "displayOnCreate" set to TRUE. In practice, only the API is affected because the UI has special logic that already shows these fields when datasets are created. See also and #10850 and #10884. ### Feature Flags Can Be Listed From 7be0b678612cbd1e1735717c917d4bd3cd19146e Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 11:28:46 -0400 Subject: [PATCH 35/44] fixed update-fields.sh url (it had "9.4.1" in it; and we probably don't want to get it from the master branch either) --- doc/release-notes/6.4-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 4268f6fb56e..7ef1aa9fae1 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -446,7 +446,7 @@ service solr start Before starting Solr, update the schema to include all the extra metadata fields that your installation uses. We do this by collecting the output of the Dataverse schema API and feeding it to the `update-fields.sh` script that we supply, as in the example below (modify the command lines as needed to reflect the names of the directories, if different): ```shell - wget https://raw.githubusercontent.com/IQSS/dataverse/master/conf/solr/9.4.1/update-fields.sh + wget https://raw.githubusercontent.com/IQSS/dataverse/v6.4/conf/solr/update-fields.sh chmod +x update-fields.sh curl "http://localhost:8080/api/admin/index/solr/schema" | ./update-fields.sh /usr/local/solr/solr-9.4.1/server/solr/collection1/conf/schema.xml ``` From e8e5127fdbbe57f455c8a934d8d54fec370cd39d Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 12:10:48 -0400 Subject: [PATCH 36/44] reindex instruction --- doc/release-notes/6.4-release-notes.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 7ef1aa9fae1..0bce6ee1b2c 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -455,12 +455,14 @@ Now start Solr. 8\. Reindex Solr -Below is the simple way to reindex Solr. If you have a large installation of Dataverse, you might want to reindex in place, as described in [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html). +Below is the simplest way to reindex Solr: ```shell curl http://localhost:8080/api/admin/index ``` +The API above rebuilds the existing index "in place". If you want to be absolutely sure that your index is up-to-date and consistent, you may consider wiping it clean and reindexing everything from scratch (see [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html)). Just note that, depending on the size of your database, a full reindex may take a while and the users will be seeing incomplete search results during that window. + 9\. Run reExportAll to update dataset metadata exports This step is necessary because of changes described above for the `Datacite` and `oai_dc` export formats. From 453e1992b28c470acb664ba365b2fbf0c539da61 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 12:14:47 -0400 Subject: [PATCH 37/44] removed a superfluous command line --- doc/release-notes/6.4-release-notes.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 0bce6ee1b2c..58ea4ab016e 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -499,10 +499,6 @@ PIDs can also be updated by a superuser on a per-dataset basis using To restore any broken thumbnails caused by the bug described above, you can call the `http://localhost:8080/api/admin/clearThumbnailFailureFlag` API, which will attempt to clear the flag on all files (regardless of whether caused by this bug or some other problem with the file) or the `http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID` to clear the flag for individual files. Calling the former, batch API is recommended. -```shell -http://localhost:8080/api/admin/clearThumbnailFailureFlag/$FILE_ID -``` - 12\. PermaLinks with custom base-url If you currently use PermaLinks with a custom `base-url`: You must manually append `/citation?persistentId=` to the base URL to maintain functionality. From 0b48c048967fcd41d114936d9533906b7231f83a Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 12:39:42 -0400 Subject: [PATCH 38/44] temp dir cleanup --- doc/release-notes/6.4-release-notes.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 58ea4ab016e..e682d28512b 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -213,6 +213,22 @@ When any `ApiBlockingFilter` policy applies to a request, the JSON in the body o See also #10508, #10672 and #10722. +### Cleanup of Temp Directories + +In this release we addressed an issue where copies of files uploaded via the UI were left in one specific temp directory (`.../domain1/uploads` by default). We would like to remind all the installation admins that it is strongly recommended to have some automated (and aggressive) cleanup mechanisms in place for all the temp directories used by Dataverse. For example, at Harvard/IQSS we have the following configuration for the PrimeFaces uploads directory above: (note that, even with this fix in place, PrimeFaces will be leaving a large number of small log files in that location) + +Instead of the default location (`.../domain1/uploads`) we use a directory on a dedicated partition, outside of the filesystem where Dataverse is installed, via the following JVM option: + +``` +-Ddataverse.files.uploads=/uploads/web +``` + +and we have a dedicated cronjob that runs every 30 minutes and deletes everything older than 2 hours in that directory: + +``` +15,45 * * * * /bin/find /uploads/web/ -mmin +119 -type f -name "upload*" -exec rm -f {} \; > /dev/null 2>&1 +``` + ## API Updates ### Search API: affiliation, parentDataverseName, image_url, etc. From 3a4e7c609a2528d3dfc780809a168318ab456520 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 27 Sep 2024 13:49:40 -0400 Subject: [PATCH 39/44] typo --- doc/release-notes/6.4-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index e682d28512b..49936e3260d 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -96,7 +96,7 @@ When notification emails are sent the part of the closing that says "contact us ### Ability to Disable Automatic Thumbnail Selection -It is now possible to turn off the feature that automatically selects of one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by enabling the [feature flag](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) `dataverse.feature.disable-dataset-thumbnail-autoselect`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. See also #10820. +It is now possible to turn off the feature that automatically selects one of the image datafiles to serve as the thumbnail of the parent dataset. An admin can turn it off by enabling the [feature flag](https://guides.dataverse.org/en/6.4/installation/config.html#feature-flags) `dataverse.feature.disable-dataset-thumbnail-autoselect`. When the feature is disabled, a user can still manually pick a thumbnail image, or upload a dedicated thumbnail image. See also #10820. ### More Flexible PermaLinks From 8fc75cceba599cfc25fca61825a2394ae4de61fa Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Mon, 30 Sep 2024 16:00:24 +0200 Subject: [PATCH 40/44] docs: update release notes from #10343 --- doc/release-notes/10343-trailing-comma.md | 5 ----- doc/release-notes/6.4-release-notes.md | 6 ++++++ 2 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 doc/release-notes/10343-trailing-comma.md diff --git a/doc/release-notes/10343-trailing-comma.md b/doc/release-notes/10343-trailing-comma.md deleted file mode 100644 index 03bd18715d7..00000000000 --- a/doc/release-notes/10343-trailing-comma.md +++ /dev/null @@ -1,5 +0,0 @@ -### Trailing commas in author name now permitted - -When an author name ends on a comma (e.g. "Smith,") a dataset cannot be properly loaded when using json-ld. A null check fixes this. - -For more information, see #10343. \ No newline at end of file diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index 49936e3260d..cb34c6d13df 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -229,6 +229,12 @@ and we have a dedicated cronjob that runs every 30 minutes and deletes everythin 15,45 * * * * /bin/find /uploads/web/ -mmin +119 -type f -name "upload*" -exec rm -f {} \; > /dev/null 2>&1 ``` +### Trailing commas in author name now permitted + +When an author name ends on a comma (e.g. `Smith,` or `Smith, `), the dataset page display was broken after publishing. +Instead, an "Error 500" page was presented to the user. +The underlying issue causing the JSON-LD schema.org output on the page to break was fixed in #10343. + ## API Updates ### Search API: affiliation, parentDataverseName, image_url, etc. From c3aeeb0367a5080d8fce822fbe3e0b7ead22099c Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 30 Sep 2024 10:16:00 -0400 Subject: [PATCH 41/44] tweaks #10343 --- doc/release-notes/6.4-release-notes.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/doc/release-notes/6.4-release-notes.md b/doc/release-notes/6.4-release-notes.md index cb34c6d13df..979fd16bf9e 100644 --- a/doc/release-notes/6.4-release-notes.md +++ b/doc/release-notes/6.4-release-notes.md @@ -213,27 +213,25 @@ When any `ApiBlockingFilter` policy applies to a request, the JSON in the body o See also #10508, #10672 and #10722. -### Cleanup of Temp Directories +### Cleanup of Temp Directories -In this release we addressed an issue where copies of files uploaded via the UI were left in one specific temp directory (`.../domain1/uploads` by default). We would like to remind all the installation admins that it is strongly recommended to have some automated (and aggressive) cleanup mechanisms in place for all the temp directories used by Dataverse. For example, at Harvard/IQSS we have the following configuration for the PrimeFaces uploads directory above: (note that, even with this fix in place, PrimeFaces will be leaving a large number of small log files in that location) +In this release we addressed an issue where copies of files uploaded via the UI were left in one specific temp directory (`.../domain1/uploads` by default). We would like to remind all the installation admins that it is strongly recommended to have some automated (and aggressive) cleanup mechanisms in place for all the temp directories used by Dataverse. For example, at Harvard/IQSS we have the following configuration for the PrimeFaces uploads directory above: (note that, even with this fix in place, PrimeFaces will be leaving a large number of small log files in that location) -Instead of the default location (`.../domain1/uploads`) we use a directory on a dedicated partition, outside of the filesystem where Dataverse is installed, via the following JVM option: +Instead of the default location (`.../domain1/uploads`) we use a directory on a dedicated partition, outside of the filesystem where Dataverse is installed, via the following JVM option: ``` -Ddataverse.files.uploads=/uploads/web ``` -and we have a dedicated cronjob that runs every 30 minutes and deletes everything older than 2 hours in that directory: +and we have a dedicated cronjob that runs every 30 minutes and deletes everything older than 2 hours in that directory: ``` 15,45 * * * * /bin/find /uploads/web/ -mmin +119 -type f -name "upload*" -exec rm -f {} \; > /dev/null 2>&1 ``` -### Trailing commas in author name now permitted +### Trailing Commas in Author Name Now Permitted -When an author name ends on a comma (e.g. `Smith,` or `Smith, `), the dataset page display was broken after publishing. -Instead, an "Error 500" page was presented to the user. -The underlying issue causing the JSON-LD schema.org output on the page to break was fixed in #10343. +When an author name ended in a comma (e.g. `Smith,` or `Smith, `), the dataset page was broken after publishing (a "500" error page was presented to the user). The underlying issue causing the JSON-LD Schema.org output on the page to break was fixed. See #10343 and #10776. ## API Updates @@ -483,7 +481,7 @@ Below is the simplest way to reindex Solr: curl http://localhost:8080/api/admin/index ``` -The API above rebuilds the existing index "in place". If you want to be absolutely sure that your index is up-to-date and consistent, you may consider wiping it clean and reindexing everything from scratch (see [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html)). Just note that, depending on the size of your database, a full reindex may take a while and the users will be seeing incomplete search results during that window. +The API above rebuilds the existing index "in place". If you want to be absolutely sure that your index is up-to-date and consistent, you may consider wiping it clean and reindexing everything from scratch (see [the guides](https://guides.dataverse.org/en/latest/admin/solr-search-index.html)). Just note that, depending on the size of your database, a full reindex may take a while and the users will be seeing incomplete search results during that window. 9\. Run reExportAll to update dataset metadata exports From 6bc565c15d2c7c1cfe99e4703d1683c42f69bb6f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Mon, 30 Sep 2024 12:09:30 -0400 Subject: [PATCH 42/44] post 6.4 container version --- modules/dataverse-parent/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/dataverse-parent/pom.xml b/modules/dataverse-parent/pom.xml index 76f0ddbc033..5abf2763128 100644 --- a/modules/dataverse-parent/pom.xml +++ b/modules/dataverse-parent/pom.xml @@ -446,8 +446,8 @@ Once the release has been made (tag created), change this back to "${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}" (These properties are provided by the build-helper plugin below.) --> - - ${revision} + ${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion} + From c59746d2176f1d8a7e1b567f12143175281e094a Mon Sep 17 00:00:00 2001 From: Vera Clemens <16904069+vera@users.noreply.github.com> Date: Mon, 30 Sep 2024 19:19:33 +0200 Subject: [PATCH 43/44] Importing unmanaged pids (#10805) * fix: send proper error response when trying to import dataset with unmanaged PID * test: add tests for importing datasets as JSON * docs: state that importing datasets with unmanaged PIDs is not supported --- doc/sphinx-guides/source/api/native-api.rst | 4 +- .../harvard/iq/dataverse/api/Dataverses.java | 6 + .../iq/dataverse/api/DataversesIT.java | 173 ++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 29 +++ 4 files changed, 210 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 117aceb141d..acb6131c9d2 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -887,7 +887,7 @@ Before calling the API, make sure the data files referenced by the ``POST``\ ed * This API does not cover staging files (with correct contents, checksums, sizes, etc.) in the corresponding places in the Dataverse installation's filestore. * This API endpoint does not support importing *files'* persistent identifiers. - * A Dataverse installation can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. + * A Dataverse installation can only import datasets with a valid PID that is managed by one of the PID providers that said installation is configured for. .. _import-dataset-with-type: @@ -935,7 +935,7 @@ Note that DDI XML does not have a field that corresponds to the "Subject" field .. warning:: * This API does not handle files related to the DDI file. - * A Dataverse installation can import datasets with a valid PID that uses a different protocol or authority than said server is configured for. However, the server will not update the PID metadata on subsequent update and publish actions. + * A Dataverse installation can only import datasets with a valid PID that is managed by one of the PID providers that said installation is configured for. .. _publish-dataverse-api: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java index 17e3086f184..0ee146ed99b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java @@ -407,6 +407,12 @@ public Response importDataset(@Context ContainerRequestContext crc, String jsonB if (ds.getIdentifier() == null) { return badRequest("Please provide a persistent identifier, either by including it in the JSON, or by using the pid query parameter."); } + + PidProvider pidProvider = PidUtil.getPidProvider(ds.getGlobalId().getProviderId()); + if (pidProvider == null || !pidProvider.canManagePID()) { + return badRequest("Cannot import a dataset that has a PID that doesn't match the server's settings"); + } + boolean shouldRelease = StringUtil.isTrue(releaseParam); DataverseRequest request = createDataverseRequest(u); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java index 6fbe91c8405..8c6a8244af1 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DataversesIT.java @@ -646,6 +646,179 @@ public void testImportDDI() throws IOException, InterruptedException { Response deleteUserResponse = UtilIT.deleteUser(username); assertEquals(200, deleteUserResponse.getStatusCode()); } + + @Test + public void testImport() throws IOException, InterruptedException { + + Response createUser = UtilIT.createRandomUser(); + String username = UtilIT.getUsernameFromResponse(createUser); + Response makeSuperUser = UtilIT.makeSuperUser(username); + assertEquals(200, makeSuperUser.getStatusCode()); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response publishDataverse = UtilIT.publishDataverseViaNativeApi(dataverseAlias, apiToken); + assertEquals(200, publishDataverse.getStatusCode()); + + JsonObjectBuilder datasetJson = Json.createObjectBuilder() + .add("datasetVersion", Json.createObjectBuilder() + .add("license", Json.createObjectBuilder() + .add("name", "CC0 1.0") + ) + .add("metadataBlocks", Json.createObjectBuilder() + .add("citation", Json.createObjectBuilder() + .add("fields", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("typeName", "title") + .add("value", "Test Dataset") + .add("typeClass", "primitive") + .add("multiple", false) + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("authorName", + Json.createObjectBuilder() + .add("value", "Simpson, Homer") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "authorName")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "author") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("datasetContactEmail", + Json.createObjectBuilder() + .add("value", "hsimpson@mailinator.com") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "datasetContactEmail")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "datasetContact") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add(Json.createObjectBuilder() + .add("dsDescriptionValue", + Json.createObjectBuilder() + .add("value", "This a test dataset.") + .add("typeClass", "primitive") + .add("multiple", false) + .add("typeName", "dsDescriptionValue")) + ) + ) + .add("typeClass", "compound") + .add("multiple", true) + .add("typeName", "dsDescription") + ) + .add(Json.createObjectBuilder() + .add("value", Json.createArrayBuilder() + .add("Other") + ) + .add("typeClass", "controlledVocabulary") + .add("multiple", true) + .add("typeName", "subject") + ) + ) + ) + )); + + String json = datasetJson.build().toString(); + + Response importJSONNoPid = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, null, "no"); + logger.info(importJSONNoPid.prettyPrint()); + assertEquals(400, importJSONNoPid.getStatusCode()); + + String body = importJSONNoPid.getBody().asString(); + String status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + String message = JsonPath.from(body).getString("message"); + assertEquals( + "Please provide a persistent identifier, either by including it in the JSON, or by using the pid query parameter.", + message + ); + + Response importJSONNoPidRelease = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, null, "yes"); + logger.info( importJSONNoPidRelease.prettyPrint()); + assertEquals(400, importJSONNoPidRelease.getStatusCode()); + + body = importJSONNoPidRelease.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + message = JsonPath.from(body).getString("message"); + assertEquals( + "Please provide a persistent identifier, either by including it in the JSON, or by using the pid query parameter.", + message + ); + + Response importJSONUnmanagedPid = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, "doi:10.5073/FK2/ABCD11", "no"); + logger.info(importJSONUnmanagedPid.prettyPrint()); + assertEquals(400, importJSONUnmanagedPid.getStatusCode()); + + body = importJSONUnmanagedPid.getBody().asString(); + status = JsonPath.from(body).getString("status"); + assertEquals("ERROR", status); + + message = JsonPath.from(body).getString("message"); + assertEquals( + "Cannot import a dataset that has a PID that doesn't match the server's settings", + message + ); + + // Under normal conditions, you shouldn't need to destroy these datasets. + // Uncomment if they're still around from a previous failed run. +// Response destroy1 = UtilIT.destroyDataset("doi:10.5072/FK2/ABCD11", apiToken); +// destroy1.prettyPrint(); +// Response destroy2 = UtilIT.destroyDataset("doi:10.5072/FK2/ABCD22", apiToken); +// destroy2.prettyPrint(); + + Response importJSONPid = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, "doi:10.5072/FK2/ABCD11", "no"); + logger.info(importJSONPid.prettyPrint()); + assertEquals(201, importJSONPid.getStatusCode()); + + Response importJSONPidRel = UtilIT.importDatasetViaNativeApi(apiToken, dataverseAlias, json, "doi:10.5072/FK2/ABCD22", "yes"); + logger.info(importJSONPidRel.prettyPrint()); + assertEquals(201, importJSONPidRel.getStatusCode()); + + Integer datasetIdInt = JsonPath.from(importJSONPid.body().asString()).getInt("data.id"); + + Response search1 = UtilIT.search("id:dataset_" + datasetIdInt + "_draft", apiToken); // santity check, can find it + search1.prettyPrint(); + search1.then().assertThat() + .body("data.total_count", CoreMatchers.is(1)) + .body("data.count_in_response", CoreMatchers.is(1)) + .body("data.items[0].name", CoreMatchers.is("Test Dataset")) + .statusCode(OK.getStatusCode()); + + //cleanup + + Response destroyDatasetResponse = UtilIT.destroyDataset(datasetIdInt, apiToken); + assertEquals(200, destroyDatasetResponse.getStatusCode()); + + Integer datasetIdIntPidRel = JsonPath.from(importJSONPidRel.body().asString()).getInt("data.id"); + Response destroyDatasetResponsePidRel = UtilIT.destroyDataset(datasetIdIntPidRel, apiToken); + assertEquals(200, destroyDatasetResponsePidRel.getStatusCode()); + + UtilIT.sleepForDeadlock(UtilIT.MAXIMUM_IMPORT_DURATION); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + assertEquals(200, deleteUserResponse.getStatusCode()); + } @Test public void testAttributesApi() throws Exception { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 4e20e8e4c33..4fbe84bcfcf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -3672,6 +3672,35 @@ static Response importDatasetDDIViaNativeApi(String apiToken, String dataverseAl return importDDI.post(postString); } + + static Response importDatasetViaNativeApi(String apiToken, String dataverseAlias, String json, String pid, String release) { + String postString = "/api/dataverses/" + dataverseAlias + "/datasets/:import"; + if (pid != null || release != null ) { + //postString = postString + "?"; + if (pid != null) { + postString = postString + "?pid=" + pid; + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "&release=" + release; + } + } else { + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "?release=" + release; + } + } + } + logger.info("Here importDatasetViaNativeApi"); + logger.info(postString); + + RequestSpecification importJSON = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .urlEncodingEnabled(false) + .body(json) + .contentType("application/json"); + + return importJSON.post(postString); + } + + static Response retrieveMyDataAsJsonString(String apiToken, String userIdentifier, ArrayList roleIds) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) From d9139effc5f37330b939e8a72eec96bde90004cf Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 30 Sep 2024 14:35:15 -0400 Subject: [PATCH 44/44] #10879 remove access request from render logic --- src/main/webapp/dataset-license-terms.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataset-license-terms.xhtml b/src/main/webapp/dataset-license-terms.xhtml index 255e63fbfc2..03173faf989 100644 --- a/src/main/webapp/dataset-license-terms.xhtml +++ b/src/main/webapp/dataset-license-terms.xhtml @@ -12,7 +12,7 @@ or !empty termsOfUseAndAccess.originalArchive or !empty termsOfUseAndAccess.availabilityStatus or !empty termsOfUseAndAccess.contactForAccess or !empty termsOfUseAndAccess.sizeOfCollection or !empty termsOfUseAndAccess.studyCompletion - or termsOfUseAndAccess.fileAccessRequest}"/> + }"/>