1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

安装依赖

pip install -U huggingface_hub

镜像站点

export HF_ENDPOINT=https://hf-mirror.com

登陆

huggingface-cli login

下载模型/数据集

huggingface-cli download --repo-type model|dataset <repo_id> [filename] --local-dir <local_dir>

  

可选：hf-transfer

pip install -U hf-transfer

export HF_HUB_ENABLE_HF_TRANSFER=1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

from huggingface_hub import snapshot_download

  

repo_id = "meta-llama/Llama-2-7b-hf"  # 模型在huggingface上的名称

local_dir = "/home/model_zoo/LLM/llama2/Llama-2-7b-hf/"  # 本地模型存储的地址

local_dir_use_symlinks = False  # 本地模型使用文件保存，而非blob形式保存

token = "XXX"  # 在hugging face上生成的 access token

  

# 如果需要代理的话

proxies = {

    'http': 'XXXX',

    'https': 'XXXX',

}

  

snapshot_download(

    repo_id=repo_id,

    local_dir=local_dir,

    local_dir_use_symlinks=local_dir_use_symlinks,

    token=token,

    proxies=proxies

)

  

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

from huggingface_hub import hf_hub_download

hf_hub_download(

    repo_id="google/pegasus-xsum",

    filename="config.json",

    revision="4d33b01d79672f27f001f6abade33f22d993b151"

)

1
2
3
4
5
6
7
8

降级requests和urllib3

pip install requests==2.27.1

pip install urllib3==1.25.11

os.environ["CURL_CA_BUNDLE"]=""