Skip to content

Commit

Permalink
Add User-Agent header to download_dataset function
Browse files Browse the repository at this point in the history
Fix the HTTP Error 403 by using urlretrieve()

Signed-off-by: Lizhen You <lyou@nvidia.com>
  • Loading branch information
Lizhen You authored and Lizhen You committed Dec 26, 2024
1 parent b3ce774 commit 35290e5
Showing 1 changed file with 5 additions and 2 deletions.
7 changes: 5 additions & 2 deletions python/cuvs_bench/cuvs_bench/get_dataset/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import os
import subprocess
import sys
from urllib.request import urlretrieve
import urllib.request


def get_dataset_path(name, ann_bench_data_path):
Expand All @@ -29,7 +29,10 @@ def get_dataset_path(name, ann_bench_data_path):
def download_dataset(url, path):
if not os.path.exists(path):
print(f"downloading {url} -> {path}...")
urlretrieve(url, path)
req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
with urllib.request.urlopen(req) as response, open(path, 'wb') as out_file:
data = response.read()
out_file.write(data)


def convert_hdf5_to_fbin(path, normalize):
Expand Down

0 comments on commit 35290e5

Please sign in to comment.