Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion studio/backend/hub/services/datasets/downloads.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
from hub.utils import download_registry
from hub.utils import inventory_scan as hf_cache_scan
from hub.utils.hf_cache_state import has_active_incomplete_blobs
from hub.utils.hf_tokens import hf_token_arg
from hub.utils.paths import (
is_valid_repo_id as _is_valid_repo_id,
resolve_cached_repo_id_case,
Expand Down Expand Up @@ -81,7 +82,7 @@ def get_dataset_snapshot_metadata_cached(
try:
from huggingface_hub import HfApi

info = HfApi(token = hf_token).dataset_info(
info = HfApi(token = hf_token_arg(hf_token)).dataset_info(
repo_id,
files_metadata = True,
timeout = _DATASET_SIZE_TIMEOUT_SECONDS,
Expand Down
15 changes: 6 additions & 9 deletions studio/backend/hub/services/datasets/formatting.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
from hub.utils import download_registry
from hub.utils.dataset_format import check_dataset_format, format_dataset_preview
from hub.utils.hf_errors import hf_error_status
from hub.utils.hf_tokens import hf_token_arg
from hub.utils.paths import (
is_valid_repo_id as _is_valid_repo_id,
resolve_dataset_path,
Expand Down Expand Up @@ -234,12 +235,10 @@ def _load_processed_hf_preview_slice(
"path": request.dataset_name,
"split": request.train_split or "train",
"download_config": DownloadConfig(local_files_only = True),
"token": hf_token_arg(hf_token),
}
if request.subset:
load_kwargs["name"] = request.subset
if hf_token:
load_kwargs["token"] = hf_token

dataset = load_dataset(**load_kwargs)
total_rows = len(dataset)
preview_slice = dataset.select(range(min(preview_size, total_rows)))
Expand Down Expand Up @@ -316,6 +315,7 @@ def check_format_response(
)
else:
preview_slice = None
request_token = hf_token_arg(hf_token)

try:
from huggingface_hub import HfApi
Expand All @@ -324,7 +324,7 @@ def check_format_response(
repo_files = api.list_repo_files(
request.dataset_name,
repo_type = "dataset",
token = hf_token or None,
token = request_token,
)
train_split = request.train_split or "train"
first_file = _select_tier1_repo_file(
Expand All @@ -339,9 +339,8 @@ def check_format_response(
"data_files": {train_split: [first_file]},
"split": train_split,
"streaming": True,
"token": request_token,
}
if hf_token:
load_kwargs["token"] = hf_token

streamed_ds = load_dataset(**load_kwargs)
rows = list(islice(streamed_ds, PREVIEW_SIZE))
Expand All @@ -361,12 +360,10 @@ def check_format_response(
"path": request.dataset_name,
"split": request.train_split or "train",
"streaming": True,
"token": request_token,
}
if request.subset:
load_kwargs["name"] = request.subset
if hf_token:
load_kwargs["token"] = hf_token

streamed_ds = load_dataset(**load_kwargs)

rows = list(islice(streamed_ds, PREVIEW_SIZE))
Expand Down
4 changes: 0 additions & 4 deletions studio/backend/hub/services/download_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,10 +75,6 @@ def spawn_worker(
env["HF_HUB_DISABLE_PROGRESS_BARS"] = "1"
env["HF_HUB_DISABLE_TELEMETRY"] = "1"
env["HF_HUB_DISABLE_XET"] = "0" if use_xet else "1"
# No token in Studio settings: fall back to the backend's own HF_TOKEN so
# private repos stay downloadable (needed while inkling repos are private).
if not hf_token:
hf_token = os.environ.get("HF_TOKEN") or None
env["HF_HUB_DISABLE_IMPLICIT_TOKEN"] = "0" if hf_token else "1"
# hf_transfer's parallel Range chunks can leave sparse partials even in
# "http" mode; disable so the worker's writer is always sequential.
Expand Down
3 changes: 2 additions & 1 deletion studio/backend/hub/services/models/cache_inventory.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from hub.schemas.inventory import ModelFormat
from hub.utils import inventory_scan as hf_cache_scan
from hub.utils import download_registry
from hub.utils.hf_tokens import hf_token_arg
from hub.utils.snapshot_filters import (
snapshot_download_blob_hashes,
snapshot_download_size,
Expand Down Expand Up @@ -73,7 +74,7 @@ def get_repo_snapshot_metadata_cached(
try:
from huggingface_hub import HfApi

info = HfApi(token = hf_token).model_info(
info = HfApi(token = hf_token_arg(hf_token)).model_info(
repo_id,
files_metadata = True,
timeout = _MODEL_METADATA_TIMEOUT_SECONDS,
Expand Down
3 changes: 2 additions & 1 deletion studio/backend/hub/services/models/gguf_variants.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
INCOMPLETE_SUFFIX,
iter_destructive_repo_cache_dirs,
)
from hub.utils.hf_tokens import hf_token_arg
from hub.utils.gguf import (
extract_quant_label,
iter_hf_cache_snapshots,
Expand Down Expand Up @@ -198,7 +199,7 @@ def _fetch_gguf_variant_requirements(
return {}
try:
from huggingface_hub import HfApi
info = HfApi(token = hf_token).model_info(
info = HfApi(token = hf_token_arg(hf_token)).model_info(
repo_id,
files_metadata = True,
timeout = _GGUF_METADATA_TIMEOUT_SECONDS,
Expand Down
75 changes: 74 additions & 1 deletion studio/backend/hub/tests/test_dataset_services.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

import asyncio
import sys
from pathlib import Path
from types import SimpleNamespace

Expand All @@ -10,7 +11,7 @@

from hub.schemas.datasets import CheckFormatRequest, LocalDatasetItem
from hub.services.datasets import cache_inventory, downloads, formatting, local
from hub.utils import download_manifest, download_registry, state_dir
from hub.utils import download_manifest, download_registry, llm_assist, state_dir


class _Upload:
Expand Down Expand Up @@ -203,6 +204,78 @@ def test_check_format_rejects_invalid_path_as_400():
assert exc_info.value.status_code == 400


@pytest.mark.parametrize(
("hf_token", "expected_token"),
((None, False), ("request-token", "request-token")),
)
def test_check_format_remote_preview_uses_only_request_hf_token(
monkeypatch, tmp_path, hf_token, expected_token
):
api_tokens = []
load_tokens = []

class _Api:
def list_repo_files(self, *_args, **kwargs):
api_tokens.append(kwargs.get("token"))
return ["train.jsonl"]

class _Dataset(list):
@classmethod
def from_list(cls, rows):
return cls(rows)

def _load_dataset(**kwargs):
load_tokens.append(kwargs.get("token"))
return _Dataset([{"text": "hello"}])

monkeypatch.setenv("HF_TOKEN", "operator-secret-token")
monkeypatch.setattr(formatting, "resolve_dataset_path", lambda _name: tmp_path / "missing")
monkeypatch.setattr(
formatting,
"check_dataset_format",
lambda *_args, **_kwargs: {
"requires_manual_mapping": True,
"detected_format": "text",
"columns": ["text"],
},
)
monkeypatch.setitem(sys.modules, "huggingface_hub", SimpleNamespace(HfApi = _Api))
monkeypatch.setitem(
sys.modules,
"datasets",
SimpleNamespace(Dataset = _Dataset, load_dataset = _load_dataset),
)

formatting.check_format_response(
CheckFormatRequest(dataset_name = "Org/Data"),
hf_token,
)

assert api_tokens == [expected_token]
assert load_tokens == [expected_token]


@pytest.mark.parametrize(
("hf_token", "expected_token"),
((None, False), ("request-token", "request-token")),
)
def test_dataset_card_uses_only_request_hf_token(monkeypatch, hf_token, expected_token):
tokens = []

class _DatasetCard:
@classmethod
def load(cls, *_args, **kwargs):
tokens.append(kwargs.get("token"))
return SimpleNamespace(text = "", data = None)

monkeypatch.setenv("HF_TOKEN", "operator-secret-token")
monkeypatch.setitem(sys.modules, "huggingface_hub", SimpleNamespace(DatasetCard = _DatasetCard))

llm_assist._fetch_hf_dataset_card("Org/Data", hf_token)

assert tokens == [expected_token]


def test_dataset_download_status_preserves_idle_shape():
status = downloads._dataset_status("Org/Data")

Expand Down
99 changes: 99 additions & 0 deletions studio/backend/hub/tests/test_model_services.py
Original file line number Diff line number Diff line change
Expand Up @@ -2746,6 +2746,50 @@ def test_hub_hf_token_header_uses_namespaced_header_only():
assert get_hf_token(None) is None


@pytest.mark.parametrize(
("hf_token", "expected_api_token"),
(
(None, False),
("request-token", "request-token"),
),
)
def test_hub_metadata_uses_only_explicit_request_token(monkeypatch, hf_token, expected_api_token):
captured_tokens = []

class _Api:
def __init__(self, *, token = None):
captured_tokens.append(token)

def model_info(self, *_args, **_kwargs):
return SimpleNamespace(siblings = [])

def dataset_info(self, *_args, **_kwargs):
return SimpleNamespace(siblings = [], private = False, gated = False)

monkeypatch.setenv("HF_TOKEN", "operator-secret-token")
monkeypatch.setitem(sys.modules, "huggingface_hub", SimpleNamespace(HfApi = _Api))

repo_suffix = "anonymous" if hf_token is None else "explicit"
cache_inventory.get_repo_snapshot_metadata_cached(
f"Security/Model-{repo_suffix}",
hf_token,
)
gguf_variants._fetch_gguf_variant_requirements(
f"Security/GGUF-{repo_suffix}",
hf_token,
)
gguf.list_gguf_variants(
f"Security/GGUF-List-{repo_suffix}",
hf_token,
)
dataset_downloads.get_dataset_snapshot_metadata_cached(
f"Security/Dataset-{repo_suffix}",
hf_token,
)

assert captured_tokens == [expected_api_token] * 4


def test_scan_folder_rejects_credential_directories(tmp_path):
sensitive_dir = tmp_path / ".ssh" / "models"
sensitive_dir.mkdir(parents = True)
Expand Down Expand Up @@ -3164,3 +3208,58 @@ def current_generation(self, _key):

assert result.state == "running"
assert result.generation == 4


@pytest.mark.parametrize(
"worker_args",
(
["--repo-id", "attacker/private-model"],
["--repo-id", "attacker/private-dataset", "--dataset"],
),
)
def test_spawn_worker_does_not_reuse_backend_hf_token(monkeypatch, worker_args):
captured = {}

class _Proc:
pass

def _fake_popen(*_args, **kwargs):
captured["env"] = kwargs["env"]
return _Proc()

monkeypatch.setenv("HF_TOKEN", "operator-secret-token")
monkeypatch.setattr(download_lifecycle.subprocess, "Popen", _fake_popen)

proc = download_lifecycle.spawn_worker(
worker_args,
None,
use_xet = False,
)

assert isinstance(proc, _Proc)
assert "HF_TOKEN" not in captured["env"]
assert captured["env"]["HF_HUB_DISABLE_IMPLICIT_TOKEN"] == "1"


def test_spawn_worker_uses_explicit_request_hf_token(monkeypatch):
captured = {}

class _Proc:
pass

def _fake_popen(*_args, **kwargs):
captured["env"] = kwargs["env"]
return _Proc()

monkeypatch.setenv("HF_TOKEN", "operator-secret-token")
monkeypatch.setattr(download_lifecycle.subprocess, "Popen", _fake_popen)

proc = download_lifecycle.spawn_worker(
["--repo-id", "user/private-model"],
"request-token",
use_xet = False,
)

assert isinstance(proc, _Proc)
assert captured["env"]["HF_TOKEN"] == "request-token"
assert captured["env"]["HF_HUB_DISABLE_IMPLICIT_TOKEN"] == "0"
4 changes: 3 additions & 1 deletion studio/backend/hub/utils/gguf.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@

from loggers import get_logger

from hub.utils.hf_tokens import hf_token_arg

logger = get_logger(__name__)
_GGUF_MODEL_INFO_TIMEOUT_SECONDS = 5.0

Expand Down Expand Up @@ -405,7 +407,7 @@ def list_gguf_variants(
return (*cached, None)

try:
info = HfApi(token = hf_token).model_info(
info = HfApi(token = hf_token_arg(hf_token)).model_info(
repo_id,
files_metadata = True,
timeout = _GGUF_MODEL_INFO_TIMEOUT_SECONDS,
Expand Down
15 changes: 15 additions & 0 deletions studio/backend/hub/utils/hf_tokens.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Hugging Face token boundary helpers."""

from __future__ import annotations

from typing import Literal, Optional

HfTokenArg = str | Literal[False]


def hf_token_arg(hf_token: Optional[str]) -> HfTokenArg:
"""Return an explicit token or disable Hugging Face's ambient credentials."""
return hf_token if hf_token else False
Comment on lines +13 to +15

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To prevent authentication failures due to accidental copy-paste errors (such as trailing newlines or spaces), it is recommended to strip leading and trailing whitespace from the Hugging Face token. Additionally, treating a token that consists only of whitespace as empty (and thus falling back to False for anonymous access) prevents invalid authentication requests.

Suggested change
def hf_token_arg(hf_token: Optional[str]) -> HfTokenArg:
"""Return an explicit token or disable Hugging Face's ambient credentials."""
return hf_token if hf_token else False
def hf_token_arg(hf_token: Optional[str]) -> HfTokenArg:
"""Return an explicit token or disable Hugging Face's ambient credentials."""
if hf_token:
stripped = hf_token.strip()
if stripped:
return stripped
return False
References
  1. Maintain consistency with existing patterns by using standard trimming for sanitization.

3 changes: 2 additions & 1 deletion studio/backend/hub/utils/llm_assist.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from loggers import get_logger

from hub.utils import download_registry
from hub.utils.hf_tokens import hf_token_arg

logger = get_logger(__name__)

Expand Down Expand Up @@ -84,7 +85,7 @@ def _fetch_hf_dataset_card(
try:
from huggingface_hub import DatasetCard

card = DatasetCard.load(dataset_name, token = hf_token)
card = DatasetCard.load(dataset_name, token = hf_token_arg(hf_token))
readme = card.text or ""
if len(readme) > README_MAX_CHARS:
cut = readme[:README_MAX_CHARS].rfind(".")
Expand Down
Loading
Loading