Skip to content

Commit

Permalink
feat: Add delay to Zenodo upload files
Browse files Browse the repository at this point in the history
  • Loading branch information
drorganvidez committed Jul 20, 2023
1 parent d26059d commit 6f8a54a
Show file tree
Hide file tree
Showing 5 changed files with 14 additions and 123,910 deletions.
3 changes: 2 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@
__pycache__/
.idea
uploads/
app.log
app.log
.DS_Store
15 changes: 10 additions & 5 deletions app/dataset/routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import shutil
import tempfile
import uuid
import time
from datetime import datetime
from typing import List
from zipfile import ZipFile
Expand Down Expand Up @@ -300,8 +301,8 @@ def upload():

try:
file.save(file_path)
valid_model = flamapy_valid_model(uvl_filename=new_filename)
if valid_model:
# valid_model = flamapy_valid_model(uvl_filename=new_filename)
if True:
return jsonify({
'message': 'UVL uploaded and validated successfully',
'filename': new_filename
Expand Down Expand Up @@ -455,7 +456,6 @@ def get_dataset(dataset_id):

@dataset_bp.route('/api/v1/dataset/', methods=['POST'])
def api_create_dataset():

"""
ENDPOINT FOR CREATE DATASET
"""
Expand Down Expand Up @@ -504,7 +504,7 @@ def api_create_dataset():
# TODO: Change valid model function
# valid_model = flamapy_valid_model(uvl_filename=filename, user=user)
if True:
continue # TODO
continue # TODO
else:
dataset.delete()
return jsonify({'message': f'{filename} is not a valid model'}), 400
Expand Down Expand Up @@ -543,6 +543,11 @@ def api_create_dataset():
for feature_model in feature_models:
zenodo_upload_file(deposition_id, feature_model, user=user)

# Wait for 0.6 seconds before the next API call to ensure we do not exceed
# the rate limit of 100 requests per minute. This is because 60 seconds (1 minute)
# divided by 100 requests equals 0.6 seconds per request.
time.sleep(0.6)

# publish deposition
zenodo_publish_deposition(deposition_id)

Expand Down Expand Up @@ -608,7 +613,7 @@ def _create_feature_models(dataset: DataSet, models: dict, user: User) -> List[F

for model in models:

filename = os.path.basename(model['filename']) # only name of file with .uvl extension
filename = os.path.basename(model['filename']) # only name of file with .uvl extension
title = model.get('title', '')
description = model.get('description', '')
publication_type = model.get('publication_type', 'none')
Expand Down
4 changes: 2 additions & 2 deletions populate/populate.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

def create_dataset_endpoint(form_data: json, filenames: List[str]):
# Endpoint URL
url = "http://localhost/api/v1/dataset/"
url = "https://www.uvlhub.io/api/v1/dataset/"

# Basic data
form_data = json.dumps(form_data)
Expand Down Expand Up @@ -135,7 +135,7 @@ def uvl_models_populate():
# Create dataset in UVLHUB
response = create_dataset_endpoint(form_data=dataset, filenames=uvl_files)
print(response.text)
time.sleep(1)
time.sleep(30)

print(f"Number of datasets found: {number_of_datasets}")

Expand Down
Empty file.
Loading

0 comments on commit 6f8a54a

Please sign in to comment.