Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cherry pick tool_helpers #8996

Merged
merged 1 commit into from
Aug 23, 2024
Merged

Cherry pick tool_helpers #8996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions model_zoo/ernie-1.0/data_tools/helpers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ using namespace std;

const int32_t LONG_SENTENCE_LEN = 512;

void build_blending_indices(py::array_t<uint8_t>& dataset_index,
void build_blending_indices(py::array_t<int16_t>& dataset_index,
py::array_t<int64_t>& dataset_sample_index,
const py::array_t<double>& weights,
const int32_t num_datasets,
Expand Down Expand Up @@ -73,7 +73,7 @@ void build_blending_indices(py::array_t<uint8_t>& dataset_index,
}

// Populate the indices.
dataset_index_ptr[sample_idx] = static_cast<uint8_t>(max_error_index);
dataset_index_ptr[sample_idx] = static_cast<int16_t>(max_error_index);
dataset_sample_index_ptr[sample_idx] = current_samples[max_error_index];

// Update the total samples.
Expand Down
15 changes: 13 additions & 2 deletions paddlenlp/data/blendable_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

import hashlib
import importlib.metadata
import os
import time

Expand Down Expand Up @@ -45,8 +46,18 @@ def __init__(self, datasets, weights, size, share_folder, *, data_cache_path=Non
# Build indicies.
def _build_indices():
start_time = time.time()
assert num_datasets < 255
dataset_index = np.zeros(self.size, dtype=np.uint8)

tool_helpers_version = importlib.metadata.version("tool_helpers")
if tool_helpers_version > "0.1.1":
assert (
num_datasets < 32767
), f"Detect num_datasets({num_datasets})>=32767. Currently, num_datasets should be less than 32767."
dataset_index = np.zeros(self.size, dtype=np.int16)
else:
assert (
num_datasets < 255
), f"Detect num_datasets:({num_datasets})>=255. When 'tool_helpers<=0.1.1', num_datasets should be less than 255. To support num_datasets greater than 255, please upgrade `tool_helpers>=0.1.2`."
dataset_index = np.zeros(self.size, dtype=np.uint8)
dataset_sample_index = np.zeros(self.size, dtype=np.int64)

from tool_helpers import helpers
Expand Down
Loading