From ff8288e7b6624667f97e67f146ab2157ebd18bd3 Mon Sep 17 00:00:00 2001 From: henrykironde Date: Thu, 26 Oct 2017 01:32:24 -0400 Subject: [PATCH 1/5] Download should not create table breed-bird-survey-50stops was not working when using download_only engine --- retriever/engines/download_only.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retriever/engines/download_only.py b/retriever/engines/download_only.py index 9c8bb80c2..e767ff6c6 100644 --- a/retriever/engines/download_only.py +++ b/retriever/engines/download_only.py @@ -119,7 +119,7 @@ def dummy_method(self, *args, **kwargs): 'auto_create_table', 'insert_data_from_url', } -remove_methods = ['insert_data_from_file', 'create_db'] +remove_methods = ['insert_data_from_file', 'create_db', "create_table"] for name, method in methods: if (name not in keep_methods and 'download' not in name and From c755ad29b436eaa5f6ebfab407256050a7aa015d Mon Sep 17 00:00:00 2001 From: henrykironde Date: Thu, 26 Oct 2017 01:33:59 -0400 Subject: [PATCH 2/5] Update version before release Update CHANGES.md with major updates since last release Update the version number in retriever_installer.iss Update setup.py Run version.py --- CHANGES.md | 52 +++++++++++++++++++++++++++++++++++++++++ retriever/_version.py | 2 +- retriever_installer.iss | 2 +- setup.py | 2 +- version.txt | 4 ++-- 5 files changed, 57 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 43ffd6b93..81a16765f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,55 @@ +# v2.1.0 + +## Major changes +- Add Python interface +- Adopting the Frictionless Data specification standards +- Add Retriever to conda +- Auto complete of Retriever commands on Unix systems + +## Minor changes + +- Add license to datasets +- Change the structure of raw data from string to list +- USe precise on travis from trusty +- Add testing on any modified dataset +- Improve memory usage in cross-tab processing +- Add capabilitiy for datasets to use custom Encoding +- Test regression using python interface + +## New datasets +- Add ant dataset and weather data to the portal dataset +- NYC TreesCount +- PREDICTS +- aquatic_animal_excretion +- biodiversity_response +- bird_migration_data +- chytr_disease_distr +- croche_vegetation_data +- dicerandra_frutescens +- flensburg_food_web +- great_basin_mammal_abundance +- macroalgal_communities +- macrocystis_variation +- marine_recruitment_data +- mediter_basin_plant_traits +- nematode_traits +- ngreatplains-flowering-dates +- portal-dev +- portal +- predator_prey_body_ratio +- predicts +- socean_diet_data +- species_exctinction_rates +- streamflow_conditions +- tree_canopy_geometries +- turtle_offspring_nesting +- Add vertnet individual datasets + vertnet_amphibians + vertnet_birds + vertnet_fishes + vertnet_mammals + vertnet_reptiles + # v2.0.0 ## Major changes diff --git a/retriever/_version.py b/retriever/_version.py index 8ac1081e8..87cc967cf 100644 --- a/retriever/_version.py +++ b/retriever/_version.py @@ -1 +1 @@ -__version__ = 'v2.1.dev' +__version__ = 'v2.1.0' diff --git a/retriever_installer.iss b/retriever_installer.iss index b84c6c120..210f3c63d 100644 --- a/retriever_installer.iss +++ b/retriever_installer.iss @@ -7,7 +7,7 @@ ; (To generate a new GUID, click Tools | Generate GUID inside the IDE.) AppId={{0467A404-8C83-42A2-8C25-0903FBD14D9C} AppName=Data Retriever -AppVersion=2.0.0 +AppVersion=2.1.0 AppPublisher=Weecology AppPublisherURL=http://data-retriever.org AppSupportURL=http://data-retriever.org diff --git a/setup.py b/setup.py index adf05e32a..bd0d24cae 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ os.system("cp hooks/pre-commit .git/hooks/pre-commit") os.system("chmod +x .git/hooks/pre-commit") -__version__ = 'v2.1.dev' +__version__ = 'v2.1.0' with open(os.path.join("retriever", "_version.py"), "w") as version_file: version_file.write("__version__ = " + "'" + __version__ + "'\n") version_file.close() diff --git a/version.txt b/version.txt index 0144f80b4..56bf137f9 100644 --- a/version.txt +++ b/version.txt @@ -1,4 +1,4 @@ -v2.1.dev +v2.1.0 abalone_age.json,1.2.1 amniote_life_hist.py,2.0.2 antarctic_breed_bird.json,1.2.1 @@ -8,7 +8,7 @@ biodiversity_response.json,1.0.1 biomass_allometry_db.py,1.4.2 bird_migration_data.json,1.0.1 bird_size.json,1.2.2 -breast_cancer_wi.json,1.1.1 +breast_cancer_wi.json,1.2.1 breed_bird_survey.py,1.4.1 breed_bird_survey_50stop.py,1.4.1 butterfly_population_network.json,1.2.1 From 732c5e996c5144d3fb2a5fe60d864a8a29927b10 Mon Sep 17 00:00:00 2001 From: henrykironde Date: Thu, 26 Oct 2017 16:56:12 -0400 Subject: [PATCH 3/5] Make sure scripts are updated for new source installs. For binary installer where there is no scripts dir, and there is no ./retriever/script or ./retriever/script dir is empty, Update For the python interface, if we have not scripts detected we shall get updates --- retriever/__main__.py | 6 +++++- retriever/lib/install.py | 4 ++++ setup.py | 6 +++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/retriever/__main__.py b/retriever/__main__.py index 1f9eb5209..56f8bf992 100644 --- a/retriever/__main__.py +++ b/retriever/__main__.py @@ -17,7 +17,7 @@ from retriever.engines import engine_list, choose_engine from retriever.lib.datapackage import create_json, edit_json, delete_json, get_script_filename from retriever.lib.datasets import datasets, dataset_names, license -from retriever.lib.defaults import sample_script, CITATION, ENCODING +from retriever.lib.defaults import sample_script, CITATION, ENCODING, SCRIPT_SEARCH_PATHS from retriever.lib.get_opts import parser from retriever.lib.repository import check_for_updates from retriever.lib.scripts import SCRIPT_LIST @@ -40,6 +40,10 @@ def main(): else: # otherwise, parse them + if not os.path.isdir(SCRIPT_SEARCH_PATHS[1]) and not \ + [f for f in os.listdir(SCRIPT_SEARCH_PATHS[-1]) + if os.path.exists(SCRIPT_SEARCH_PATHS[-1])]: + check_for_updates() script_list = SCRIPT_LIST() args = parser.parse_args() diff --git a/retriever/lib/install.py b/retriever/lib/install.py index 3e4d22448..742266626 100644 --- a/retriever/lib/install.py +++ b/retriever/lib/install.py @@ -7,6 +7,7 @@ from retriever.lib.defaults import DATA_DIR from retriever.lib.scripts import SCRIPT_LIST from retriever.lib.tools import name_matches +from retriever.lib.repository import check_for_updates def _install(args, use_cache, debug): @@ -15,6 +16,9 @@ def _install(args, use_cache, debug): engine.use_cache = use_cache script_list = SCRIPT_LIST() + if not script_list: + check_for_updates() + script_list = SCRIPT_LIST() data_sets_scripts = name_matches(script_list, args['dataset']) if data_sets_scripts: for data_sets_script in data_sets_scripts: diff --git a/setup.py b/setup.py index bd0d24cae..298f0549b 100644 --- a/setup.py +++ b/setup.py @@ -16,6 +16,10 @@ os.system("cp hooks/pre-commit .git/hooks/pre-commit") os.system("chmod +x .git/hooks/pre-commit") +app_data = "~/.retriever/scripts" +if os.path.exists(app_data): + os.system("rm -r {}".format(app_data)) + __version__ = 'v2.1.0' with open(os.path.join("retriever", "_version.py"), "w") as version_file: version_file.write("__version__ = " + "'" + __version__ + "'\n") @@ -105,7 +109,7 @@ def clean_version(v): from retriever.compile import compile from retriever.lib.repository import check_for_updates - compile() check_for_updates(False) + compile() except: pass From 4e872071d2a5511726cc2b2e9cf6abae20856c77 Mon Sep 17 00:00:00 2001 From: henrykironde Date: Thu, 26 Oct 2017 18:03:53 -0400 Subject: [PATCH 4/5] Correct CHANGES.md mistakes --- CHANGES.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 81a16765f..71c2a17ea 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -2,7 +2,6 @@ ## Major changes - Add Python interface -- Adopting the Frictionless Data specification standards - Add Retriever to conda - Auto complete of Retriever commands on Unix systems @@ -10,11 +9,11 @@ - Add license to datasets - Change the structure of raw data from string to list -- USe precise on travis from trusty - Add testing on any modified dataset - Improve memory usage in cross-tab processing - Add capabilitiy for datasets to use custom Encoding -- Test regression using python interface +- Use new Python interface for regression testing +- Use Frictionless Data specification terminology for internals ## New datasets - Add ant dataset and weather data to the portal dataset From 8acfed87fbcd084920054d27f485b37bbd779537 Mon Sep 17 00:00:00 2001 From: henrykironde Date: Thu, 26 Oct 2017 18:47:37 -0400 Subject: [PATCH 5/5] Add checks for python interface functions. This catches some edge cases when no scripts --- retriever/lib/download.py | 11 ++++++++--- retriever/lib/install.py | 6 +++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/retriever/lib/download.py b/retriever/lib/download.py index ce9463883..f81c66b15 100644 --- a/retriever/lib/download.py +++ b/retriever/lib/download.py @@ -1,11 +1,13 @@ from __future__ import absolute_import from __future__ import print_function +import os + from retriever.engines import choose_engine +from retriever.lib.defaults import DATA_DIR, SCRIPT_WRITE_PATH from retriever.lib.scripts import SCRIPT_LIST from retriever.lib.tools import name_matches - -script_list = SCRIPT_LIST() +from retriever.lib.repository import check_for_updates def download(dataset, path='./', quiet=False, subdir=False, debug=False): @@ -18,7 +20,10 @@ def download(dataset, path='./', quiet=False, subdir=False, debug=False): 'quiet': quiet } engine = choose_engine(args) - + script_list = SCRIPT_LIST() + if not script_list or not os.listdir(SCRIPT_WRITE_PATH): + check_for_updates() + script_list = SCRIPT_LIST(force_compile=False) scripts = name_matches(script_list, args['dataset']) if scripts: for script in scripts: diff --git a/retriever/lib/install.py b/retriever/lib/install.py index 742266626..14bc8f7aa 100644 --- a/retriever/lib/install.py +++ b/retriever/lib/install.py @@ -4,7 +4,7 @@ import os from retriever.engines import choose_engine -from retriever.lib.defaults import DATA_DIR +from retriever.lib.defaults import DATA_DIR, SCRIPT_WRITE_PATH from retriever.lib.scripts import SCRIPT_LIST from retriever.lib.tools import name_matches from retriever.lib.repository import check_for_updates @@ -16,9 +16,9 @@ def _install(args, use_cache, debug): engine.use_cache = use_cache script_list = SCRIPT_LIST() - if not script_list: + if not script_list or not os.listdir(SCRIPT_WRITE_PATH): check_for_updates() - script_list = SCRIPT_LIST() + script_list = SCRIPT_LIST(force_compile=False) data_sets_scripts = name_matches(script_list, args['dataset']) if data_sets_scripts: for data_sets_script in data_sets_scripts: