图像数据集

import os
from pathlib import Path

# os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"

from huggingface_hub import HfApi, logging

repo_id = "PixArt-alpha/SAM-LLaVA-Captions10M"
repo_type = "dataset"

local_name = repo_id.split("/")[1]

logging.set_verbosity_debug()
hf = HfApi()
hf.snapshot_download(
    repo_id=repo_id,
    repo_type=repo_type,
    # revision="refs/convert/parquet",
    local_dir=Path("./") / local_name,
    local_dir_use_symlinks=False,
    resume_download=True,
    cache_dir="./cache",
    max_workers=16,
)

import aiohttp
import asyncio
import json


async def fetch(session, url):
    async with session.get(url) as response:
        return await response.text()


async def main():

    repo_id = "PixArt-alpha/PixArt-alpha"
    repo_type = "model"
    urls = []

    type_route_api = "models/"
    type_route_dl = ""
    if repo_type == "dataset":
        type_route_api = "datasets/"
        type_route_dl = "datasets/"

    url = f'https://hf-mirror.com/api/{type_route_api}{repo_id}'
    async with aiohttp.ClientSession() as session:
        response_text = await fetch(session, url)
        response_json = json.loads(response_text)

        for sibling in response_json["siblings"]:
            download_url = f"https://hf-mirror.com/{type_route_dl}{repo_id}/resolve/main/" + sibling["rfilename"] + "?download=true"
            urls.append(download_url)

    with open('output_urls.txt', 'w') as file:
        for item in urls:
            file.write(item + '\n')

loop = asyncio.get_event_loop()
loop.run_until_complete(main())