From 15e6c76ad698ca180ed2ef2a8323b7d1dc09733e Mon Sep 17 00:00:00 2001 From: Sam Maurer Date: Tue, 19 Feb 2019 14:52:53 -0800 Subject: [PATCH] Better path normalization --- tests/test_tables.py | 25 +++++++------------- urbansim_templates/io/tables.py | 42 +++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/tests/test_tables.py b/tests/test_tables.py index 940dcc9..e084e74 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -218,6 +218,7 @@ def test_csv(orca_session, data): modelmanager.register(t) assert 'buildings' in orca.list_tables() + _ = orca.get_table('buildings').to_frame() modelmanager.initialize() assert 'buildings' in orca.list_tables() @@ -239,6 +240,7 @@ def test_hdf(orca_session, data): modelmanager.register(t) assert 'buildings' in orca.list_tables() + _ = orca.get_table('buildings').to_frame() modelmanager.initialize() assert 'buildings' in orca.list_tables() @@ -262,6 +264,7 @@ def test_extra_settings(orca_session, data): modelmanager.register(t) assert 'buildings' in orca.list_tables() + _ = orca.get_table('buildings').to_frame() modelmanager.initialize() assert 'buildings' in orca.list_tables() @@ -269,26 +272,14 @@ def test_extra_settings(orca_session, data): modelmanager.remove_step('buildings') -def test_windows_path(orca_session, data): +def test_windows_paths(orca_session, data): """ - Test loading a file with Windows-formatted path. + Test in Windows that a Windows-style path is properly normalized. - """ - t = TableFromDisk() - t.name = 'buildings' - t.source_type = 'csv' - t.path = 'data\buildings.csv' - t.csv_index_cols = 'building_id' - - assert 'buildings' not in orca.list_tables() - - modelmanager.register(t) - assert 'buildings' in orca.list_tables() - - modelmanager.initialize() - assert 'buildings' in orca.list_tables() + TO DO - implement - modelmanager.remove_step('buildings') + """ + pass def test_without_autorun(orca_session, data): diff --git a/urbansim_templates/io/tables.py b/urbansim_templates/io/tables.py index 6ffd872..87982d8 100644 --- a/urbansim_templates/io/tables.py +++ b/urbansim_templates/io/tables.py @@ -1,6 +1,9 @@ from __future__ import print_function -import os +try: + import pathlib # Python 3.4+ +except: + import os import orca import pandas as pd @@ -54,9 +57,11 @@ class TableFromDisk(): path : str, optional Local file path to load data from, either absolute or relative to the - ModelManager config directory. Path will be normed at runtime using - `os.path.normpath()`, so either a Unix-style or Windows-style path should work - across platforms. + ModelManager config directory. The string you provide will immediately be + normalized to a platform-agnostic format, using `os.path.normpath()` in Python 2 + or `pathlib.Path()` in Python 3. It is always safe to provide a Unix-style path, + and you may provide a Windows-style path if you are creating the model step in + Windows. Saved steps will run on any platform. url : str, optional - NOT YET IMPLEMENTED Remote url to download file from. @@ -76,15 +81,15 @@ class TableFromDisk(): orca_test_spec : dict, optional - NOT YET IMPLEMENTED Data characteristics to be tested when the table is validated. - cache : bool, optional + cache : bool, default True Passed to `orca.table()`. Note that the default is `True`, unlike in the underlying general-purpose Orca function, because tables read from disk should not need to be regenerated during the course of a model run. - cache_scope : 'step', 'iteration', or 'forever', optional + cache_scope : 'step', 'iteration', or 'forever', default 'forever' Passed to `orca.table()`. Default is 'forever', as in Orca. - copy_col : bool, optional + copy_col : bool, default True Passed to `orca.table()`. Default is `True`, as in Orca. name : str, optional @@ -94,7 +99,7 @@ class TableFromDisk(): tags : list of str, optional Tags, passed to ModelManager. - autorun : bool, optional (default True) + autorun : bool, default True Automatically run the step whenever it's registered with ModelManager. Properties and attributes @@ -107,7 +112,7 @@ def __init__(self, source_type = None, path = None, csv_index_cols = None, - extra_settings = None, + extra_settings = {}, cache = True, cache_scope = 'forever', copy_col = True, @@ -189,6 +194,20 @@ def to_dict(self): return d + @property + def path(self): + return self.__path + @path.setter + def path(self, value): + if value is not None: + try: + value = str(pathlib.Path(value)) # Python 3.4+ + except: + value = os.path.normpath(value) + self.__path = value + print(value) + + def run(self): """ Register a data table with Orca. @@ -210,7 +229,6 @@ def run(self): if self.path is None: raise ValueError("Please provide a file path") - path = os.path.normpath(self.path) kwargs = self.extra_settings # Table from CSV file @@ -223,7 +241,7 @@ def run(self): cache_scope = self.cache_scope, copy_col = self.copy_col) def orca_table(): - df = pd.read_csv(path, **kwargs).set_index(self.csv_index_cols) + df = pd.read_csv(self.path, **kwargs).set_index(self.csv_index_cols) return df # Table from HDF file @@ -233,7 +251,7 @@ def orca_table(): cache_scope = self.cache_scope, copy_col = self.copy_col) def orca_table(): - df = pd.read_hdf(path, **kwargs) + df = pd.read_hdf(self.path, **kwargs) return df