[Bugfix][Omni-DuplexEval] Load local Hugging Face dataset layouts in duplex eval loader (#7331)
Signed-off-by: RyanYun09 <RyanYun09@users.noreply.github.com> Co-authored-by: RyanYun09 <RyanYun09@users.noreply.github.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
5 files changed
+658
-37
No files matched your search
@@ -64,6 +64,177 @@ def test_hugging_face_subset_names_are_splits(monkeypatch):
|
||||
assert sample.task_type == "correction"
|
||||
|
||||
|
||||
def test_local_hf_dataset_directory_is_routed_to_hf_loader(tmp_path, monkeypatch):
|
||||
# An existing local Hugging Face dataset layout (directory with a
|
||||
# ``data/*.parquet`` file) must be loaded through ``datasets.load_dataset``
|
||||
# instead of being parsed as a JSON/JSONL manifest.
|
||||
hf_dir = tmp_path / "Omni-DuplexEval"
|
||||
data_dir = hf_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
(data_dir / "train-00000-of-00001.parquet").write_bytes(b"PAR1")
|
||||
calls = []
|
||||
|
||||
def fake_load_dataset(name, **kwargs):
|
||||
calls.append((name, kwargs))
|
||||
return [{"id": "pr", "split": "PR_correction", "question_text": "Correct this."}]
|
||||
|
||||
monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=fake_load_dataset))
|
||||
samples = load_samples(hf_dir)
|
||||
assert calls == [(str(hf_dir), {})]
|
||||
assert len(samples) == 1
|
||||
assert samples[0].task_type == "correction"
|
||||
|
||||
|
||||
def test_local_parquet_file_is_routed_to_hf_loader(tmp_path, monkeypatch):
|
||||
parquet = tmp_path / "samples.parquet"
|
||||
parquet.write_bytes(b"PAR1")
|
||||
calls = []
|
||||
|
||||
def fake_load_dataset(name, **kwargs):
|
||||
calls.append((name, kwargs))
|
||||
return [{"id": "pr", "split": "PR_correction", "question_text": "Correct this."}]
|
||||
|
||||
monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=fake_load_dataset))
|
||||
samples = load_samples(str(parquet))
|
||||
assert calls == [("parquet", {"data_files": str(parquet)})]
|
||||
assert len(samples) == 1
|
||||
assert samples[0].family == "pr"
|
||||
|
||||
|
||||
def test_hf_collapse_to_train_keeps_row_split_identity(tmp_path, monkeypatch):
|
||||
# datasets.load_dataset collapses a config-less local mirror (a bare data/
|
||||
# directory) into a single generic ``train`` split. Rows that already carry
|
||||
# their own split/subset identity must keep it so family inference is not
|
||||
# poisoned by the generic name (regression for the direct-read path).
|
||||
hf_dir = tmp_path / "Omni-DuplexEval"
|
||||
data_dir = hf_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
(data_dir / "train-00000-of-00021.parquet").write_bytes(b"PAR1")
|
||||
|
||||
def fake_load_dataset(name, **kwargs):
|
||||
return {
|
||||
"train": [
|
||||
{"id": "pr", "split": "PR_correction", "question_text": "Correct this."},
|
||||
{"id": "pr-subset", "subset": "PR_event_reminder", "question_text": "Remind me."},
|
||||
{"id": "rtd", "split": "RTD_OCR", "question_text": "Read this."},
|
||||
]
|
||||
}
|
||||
|
||||
monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=fake_load_dataset))
|
||||
samples = load_samples(hf_dir)
|
||||
by_id = {sample.id: sample for sample in samples}
|
||||
assert by_id["pr"].split == "PR_correction"
|
||||
assert by_id["pr"].family == "pr"
|
||||
assert by_id["pr"].task_type == "correction"
|
||||
assert by_id["pr-subset"].split == "PR_event_reminder"
|
||||
assert by_id["pr-subset"].family == "pr"
|
||||
assert by_id["pr-subset"].task_type == "proactive_reminder"
|
||||
assert by_id["rtd"].split == "RTD_OCR"
|
||||
assert by_id["rtd"].family == "rtd"
|
||||
|
||||
|
||||
def test_hf_config_layout_stamps_split_name_when_row_has_none(tmp_path, monkeypatch):
|
||||
# With a proper config-per-split mirror the loader still stamps the Hugging
|
||||
# Face split name onto rows that do not identify themselves (unchanged).
|
||||
hf_dir = tmp_path / "Omni-DuplexEval"
|
||||
data_dir = hf_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
(data_dir / "RTD_OCR-00000-of-00001.parquet").write_bytes(b"PAR1")
|
||||
(data_dir / "PR_correction-00000-of-00001.parquet").write_bytes(b"PAR1")
|
||||
|
||||
def fake_load_dataset(name, **kwargs):
|
||||
return {
|
||||
"RTD_OCR": [{"id": "rtd", "question_text": "Read this."}],
|
||||
"PR_correction": [{"id": "pr", "question_text": "Correct this."}],
|
||||
}
|
||||
|
||||
monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=fake_load_dataset))
|
||||
samples = load_samples(hf_dir)
|
||||
by_id = {sample.id: sample for sample in samples}
|
||||
assert by_id["rtd"].split == "RTD_OCR"
|
||||
assert by_id["rtd"].family == "rtd"
|
||||
assert by_id["pr"].split == "PR_correction"
|
||||
assert by_id["pr"].family == "pr"
|
||||
assert by_id["pr"].task_type == "correction"
|
||||
|
||||
|
||||
def test_hf_collapse_to_train_without_row_identity_raises_actionable_error(tmp_path, monkeypatch):
|
||||
# A row that carries no split/family/task_type and lands in a generic
|
||||
# collapsed ``train`` split cannot be routed; the loader must raise a clear,
|
||||
# actionable ValueError instead of the bare family_for_split message.
|
||||
hf_dir = tmp_path / "Omni-DuplexEval"
|
||||
data_dir = hf_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
(data_dir / "train-00000-of-00021.parquet").write_bytes(b"PAR1")
|
||||
|
||||
def fake_load_dataset(name, **kwargs):
|
||||
return {"train": [{"id": "x", "question_text": "What is this?"}]}
|
||||
|
||||
monkeypatch.setitem(sys.modules, "datasets", SimpleNamespace(load_dataset=fake_load_dataset))
|
||||
with pytest.raises(ValueError, match="Hugging Face splits"):
|
||||
load_samples(hf_dir)
|
||||
|
||||
|
||||
def test_non_manifest_file_raises_clear_error(tmp_path):
|
||||
# A plain non-JSON file (e.g. a README) must surface a clear ValueError
|
||||
# listing the accepted inputs instead of a raw JSONDecodeError.
|
||||
bad = tmp_path / "README.md"
|
||||
bad.write_text("# readme\nnot a manifest", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="JSON/JSONL manifest"):
|
||||
load_samples(str(bad))
|
||||
|
||||
|
||||
def _write_parquet(path, rows):
|
||||
pyarrow = pytest.importorskip("pyarrow")
|
||||
pyarrow.parquet.write_table(pyarrow.Table.from_pylist(rows), str(path))
|
||||
|
||||
|
||||
def test_local_parquet_split_filters_rows_by_identity(tmp_path):
|
||||
# Regression: a single ``.parquet`` file is exposed by ``load_dataset`` as a
|
||||
# generic ``train`` physical split, so ``--split RTD_OCR`` used to raise
|
||||
# ``Unknown split "RTD_OCR"``. The requested split must instead filter the
|
||||
# rows by their preserved identity.
|
||||
pytest.importorskip("datasets")
|
||||
parquet = tmp_path / "samples.parquet"
|
||||
_write_parquet(
|
||||
parquet,
|
||||
[
|
||||
{"id": "rtd", "split": "RTD_OCR", "question_text": "Read this."},
|
||||
{"id": "pr", "split": "PR_correction", "question_text": "Correct this."},
|
||||
],
|
||||
)
|
||||
assert {sample.id for sample in load_samples(str(parquet))} == {"rtd", "pr"}
|
||||
rtd = load_samples(str(parquet), split="RTD_OCR")
|
||||
assert [sample.id for sample in rtd] == ["rtd"]
|
||||
assert rtd[0].family == "rtd"
|
||||
pr = load_samples(str(parquet), split="PR_correction")
|
||||
assert [sample.id for sample in pr] == ["pr"]
|
||||
assert pr[0].family == "pr"
|
||||
assert pr[0].task_type == "correction"
|
||||
|
||||
|
||||
def test_local_directory_named_splits_resolve_without_row_identity(tmp_path):
|
||||
# Regression: a single-configuration mirror whose data files are named after
|
||||
# the benchmark splits must resolve each split for real (no fake loader) and
|
||||
# without any per-row identity.
|
||||
pytest.importorskip("datasets")
|
||||
hf_dir = tmp_path / "Omni-DuplexEval"
|
||||
data_dir = hf_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
_write_parquet(data_dir / "RTD_OCR-00000-of-00001.parquet", [{"id": "rtd", "question_text": "Read this."}])
|
||||
_write_parquet(
|
||||
data_dir / "PR_correction-00000-of-00001.parquet",
|
||||
[{"id": "pr", "question_text": "Correct this."}],
|
||||
)
|
||||
by_id = {sample.id: sample for sample in load_samples(hf_dir)}
|
||||
assert by_id["rtd"].split == "RTD_OCR"
|
||||
assert by_id["rtd"].family == "rtd"
|
||||
assert by_id["pr"].split == "PR_correction"
|
||||
assert by_id["pr"].family == "pr"
|
||||
assert by_id["pr"].task_type == "correction"
|
||||
assert [sample.id for sample in load_samples(hf_dir, split="RTD_OCR")] == ["rtd"]
|
||||
|
||||
|
||||
def test_response_aliases_and_clock_guard():
|
||||
assert split_text("One. Two!") == ["One.", "Two!"]
|
||||
assert normalize_response_items({"chunks": [{"text": "x", "current_time": 800}]}) == [
|
||||
@@ -141,3 +312,113 @@ def test_frame_sample_content_passes_frames(tmp_path, monkeypatch):
|
||||
judge_video_mode="frame-sample",
|
||||
)
|
||||
assert score["content"]["frame_count"] == 2
|
||||
|
||||
|
||||
def test_local_directory_collapsed_train_with_row_identity_and_split_filter(tmp_path):
|
||||
# Reviewer-pointed gap: a collapsed local mirror (data/train-*.parquet)
|
||||
# whose rows carry their own split identity must correctly filter by
|
||||
# --split RTD_OCR.
|
||||
pytest.importorskip("datasets")
|
||||
hf_dir = tmp_path / "Omni-DuplexEval"
|
||||
data_dir = hf_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
_write_parquet(
|
||||
data_dir / "train-00000-of-00001.parquet",
|
||||
[
|
||||
{"id": "rtd", "split": "RTD_OCR", "question_text": "Read this."},
|
||||
{"id": "pr", "split": "PR_correction", "question_text": "Correct this."},
|
||||
],
|
||||
)
|
||||
all_samples = load_samples(hf_dir)
|
||||
assert {s.id for s in all_samples} == {"rtd", "pr"}
|
||||
rtd = load_samples(hf_dir, split="RTD_OCR")
|
||||
assert [s.id for s in rtd] == ["rtd"]
|
||||
assert rtd[0].family == "rtd"
|
||||
|
||||
|
||||
def test_local_directory_mistyped_split_raises_clear_error(tmp_path):
|
||||
# A named-shard mirror with --split RTD_TYPO must raise ValueError
|
||||
# listing the observed splits instead of silently returning nothing.
|
||||
pytest.importorskip("datasets")
|
||||
hf_dir = tmp_path / "Omni-DuplexEval"
|
||||
data_dir = hf_dir / "data"
|
||||
data_dir.mkdir(parents=True)
|
||||
_write_parquet(
|
||||
data_dir / "RTD_OCR-00000-of-00001.parquet",
|
||||
[{"id": "rtd", "question_text": "Read this."}],
|
||||
)
|
||||
_write_parquet(
|
||||
data_dir / "PR_correction-00000-of-00001.parquet",
|
||||
[{"id": "pr", "question_text": "Correct this."}],
|
||||
)
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
load_samples(hf_dir, split="RTD_TYPO")
|
||||
message = str(exc_info.value)
|
||||
assert "matched zero" in message
|
||||
assert "RTD_OCR" in message or "PR_correction" in message
|
||||
|
||||
|
||||
def test_manifest_without_split_identity_split_override_still_works(tmp_path):
|
||||
# Manifest rows without split/subset/config identity must keep the
|
||||
# base override behaviour: --split RTD_OCR stamps the split so rows
|
||||
# are kept instead of rejected (regression test for P2).
|
||||
manifest = tmp_path / "samples.json"
|
||||
manifest.write_text(
|
||||
json.dumps([{"id": "a", "video": "clip.mp4"}, {"id": "b", "video": "clip2.mp4"}]),
|
||||
encoding="utf-8",
|
||||
)
|
||||
samples = load_samples(manifest, split="RTD_OCR")
|
||||
assert len(samples) == 2
|
||||
assert all(s.split == "RTD_OCR" for s in samples)
|
||||
assert all(s.family == "rtd" for s in samples)
|
||||
|
||||
|
||||
def test_iterable_without_split_identity_split_override_still_works():
|
||||
# Iterable rows without split/subset/config identity must keep the
|
||||
# base override behaviour (regression test for F5/P2).
|
||||
data = [{"id": "a", "video": "clip.mp4"}, {"id": "b", "video": "clip2.mp4"}]
|
||||
samples = load_samples(data, split="RTD_OCR")
|
||||
assert len(samples) == 2
|
||||
assert all(s.split == "RTD_OCR" for s in samples)
|
||||
assert all(s.family == "rtd" for s in samples)
|
||||
|
||||
|
||||
def test_audio_feature_decode_false_preserves_bytes_key(tmp_path):
|
||||
# When a parquet file carries an Audio feature column, the loader
|
||||
# must recast it with decode=False so rows contain the ``bytes`` key
|
||||
# instead of ``array``/``sampling_rate``.
|
||||
pytest.importorskip("datasets")
|
||||
pytest.importorskip("pyarrow")
|
||||
import datasets
|
||||
|
||||
audio_bytes = b"\x00\x01\x02"
|
||||
ds = datasets.Dataset.from_dict(
|
||||
{
|
||||
"id": ["a"],
|
||||
"question_text": ["Hello"],
|
||||
"task_type": ["pr_correction"],
|
||||
"audio": [{"bytes": audio_bytes, "path": None}],
|
||||
},
|
||||
features=datasets.Features(
|
||||
{
|
||||
"id": datasets.Value("string"),
|
||||
"question_text": datasets.Value("string"),
|
||||
"task_type": datasets.Value("string"),
|
||||
"audio": datasets.Audio(sampling_rate=16000),
|
||||
}
|
||||
),
|
||||
)
|
||||
parquet = tmp_path / "audio_sample.parquet"
|
||||
ds.to_parquet(str(parquet))
|
||||
|
||||
samples = load_samples(str(parquet))
|
||||
assert len(samples) == 1
|
||||
raw = samples[0].raw
|
||||
assert raw is not None
|
||||
audio_val = raw.get("audio")
|
||||
assert audio_val is not None
|
||||
# The Audio feature was decoded with decode=False, so the struct
|
||||
# keeps the ``bytes`` key that materialize_media needs.
|
||||
assert isinstance(audio_val, dict) and "bytes" in audio_val
|
||||
assert audio_val["bytes"] == audio_bytes
|
||||
assert "path" in audio_val
|
||||
Reference in new issue
Block a user