From 508ca382f196b429c219d28207447b5a429eb0c9 Mon Sep 17 00:00:00 2001 From: kzahiri1 Date: Mon, 5 Oct 2026 16:20:22 -0700 Subject: [PATCH] fix: raise MAX_FILE_SIZE_GB to the documented 100 GB limit check_file still rejected files above 50.1 GB even though the fine-tuning docs advertise 100 GB. Co-authored-by: Cursor --- src/together/lib/constants.py | 2 +- tests/unit/test_files_checks.py | 47 +++++++++++++++++++++ tests/unit/test_multipart_upload_manager.py | 2 +- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/src/together/lib/constants.py b/src/together/lib/constants.py index 28eebf400..6765d6cb1 100644 --- a/src/together/lib/constants.py +++ b/src/together/lib/constants.py @@ -35,7 +35,7 @@ NUM_BYTES_IN_GB = 2**30 # maximum number of GB sized files we support finetuning for -MAX_FILE_SIZE_GB = 50.1 +MAX_FILE_SIZE_GB = 100 # Multimodal limits MAX_IMAGES_PER_EXAMPLE = 10 diff --git a/tests/unit/test_files_checks.py b/tests/unit/test_files_checks.py index 4f9538a1e..2d632c50d 100644 --- a/tests/unit/test_files_checks.py +++ b/tests/unit/test_files_checks.py @@ -1,10 +1,12 @@ import csv import json +import os from typing import Any, Dict, List, cast from pathlib import Path import pytest +from together.lib.constants import MAX_FILE_SIZE_GB, NUM_BYTES_IN_GB from together.lib.utils.files import FileCheckProgress, check_file @@ -592,6 +594,51 @@ def test_check_file_rejects_oversized_parquet_before_format_checks( assert "Maximum supported file size" in report["message"] +def _report_for_reported_file_size( + file: Path, monkeypatch: pytest.MonkeyPatch, size: int +) -> Dict[str, Any]: + real_stat = os.stat + + def controlled_stat(path, *args, **kwargs): + result = real_stat(path, *args, **kwargs) + if os.fspath(path) == file.as_posix(): + fields = list(result) + fields[6] = size + return os.stat_result(fields) + return result + + monkeypatch.setattr("together.lib.utils.files.os.stat", controlled_stat) + return check_file(file, purpose="fine-tune") + + +def test_check_file_accepts_60gb_under_documented_100gb_limit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + file = tmp_path / "tiny.jsonl" + file.write_text('{"text": "local validation fixture"}\n' * 100) + size = 60_000_000_000 + + report = _report_for_reported_file_size(file, monkeypatch, size) + + assert report["file_size"] == size + assert report["is_check_passed"] + assert report["num_samples"] == 100 + + +def test_check_file_rejects_files_above_max_size( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + file = tmp_path / "tiny.jsonl" + file.write_text('{"text": "local validation fixture"}\n') + size = int((MAX_FILE_SIZE_GB + 1) * NUM_BYTES_IN_GB) + + report = _report_for_reported_file_size(file, monkeypatch, size) + + assert not report["is_check_passed"] + assert report["file_size"] == size + assert "Maximum supported file size" in report["message"] + + def test_check_progress_tracker_does_not_reset_across_phases(tmp_path: Path) -> None: from together.lib.cli.components.check_progress import CheckProgressTracker diff --git a/tests/unit/test_multipart_upload_manager.py b/tests/unit/test_multipart_upload_manager.py index 4e8f06c9e..82aadc8c5 100644 --- a/tests/unit/test_multipart_upload_manager.py +++ b/tests/unit/test_multipart_upload_manager.py @@ -35,7 +35,7 @@ def test_calculate_parts_large_file(): @patch("together.lib.resources.files.os.stat") def test_file_size_exceeds_limit_raises_error(mock_stat: MagicMock): - """Uploading a file above 50.1GB should raise FileTypeError.""" + """Uploading a file above the configured max size should raise FileTypeError.""" mock_stat.return_value.st_size = int((MAX_FILE_SIZE_GB + 1) * NUM_BYTES_IN_GB) manager = MultipartUploadManager(MagicMock())