Merge pull request #61 from HERA-Team/py3

Upgrade to python3

.circleci/config.yml

@@ -7,7 +7,7 @@ jobs:
       python_version:
         type: string
     docker:
-      - image: continuumio/miniconda:latest
+      - image: continuumio/miniconda3:latest
       - image: circleci/postgres:11
         environment:
           POSTGRES_USER: root
@@ -60,5 +60,5 @@ workflows:
   build_and_test:
     jobs:
       - librarian:
-          name: librarian_2.7
-          python_version: "2.7"
+          name: librarian_3.7
+          python_version: "3.7"

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Version *next* (not yet released)
 
+
+# Version 1.0.0 (2019 Sep 19)
+
+- Convert codebase from python2 to python3.
 - Move scripts to `cli.py` module and replace with single command.
 - Reorganize repo structure.
 - Add tests and CI support.

alembic/env.py

@@ -6,7 +6,7 @@
 migration stuff.
 
 """
-from __future__ import with_statement
+
 
 # A hack so that we can get the librarian_server module.
 import sys

alembic/versions/71df5b41ae41_initial_schema.py

@@ -22,8 +22,8 @@ def upgrade():
     # This is the schema used by the first Librarian deployments.
     op.create_table('observing_session',
                     sa.Column('id', sa.BigInteger(), nullable=False),
-                    sa.Column('start_time_jd', sa.Float(precision=u'53'), nullable=False),
-                    sa.Column('stop_time_jd', sa.Float(precision=u'53'), nullable=False),
+                    sa.Column('start_time_jd', sa.Float(precision='53'), nullable=False),
+                    sa.Column('stop_time_jd', sa.Float(precision='53'), nullable=False),
                     sa.PrimaryKeyConstraint('id')
                     )
 
@@ -49,11 +49,11 @@ def upgrade():
 
     op.create_table('observation',
                     sa.Column('obsid', sa.BigInteger(), nullable=False),
-                    sa.Column('start_time_jd', sa.Float(precision=u'53'), nullable=False),
-                    sa.Column('stop_time_jd', sa.Float(precision=u'53'), nullable=True),
-                    sa.Column('start_lst_hr', sa.Float(precision=u'53'), nullable=True),
+                    sa.Column('start_time_jd', sa.Float(precision='53'), nullable=False),
+                    sa.Column('stop_time_jd', sa.Float(precision='53'), nullable=True),
+                    sa.Column('start_lst_hr', sa.Float(precision='53'), nullable=True),
                     sa.Column('session_id', sa.BigInteger(), nullable=True),
-                    sa.ForeignKeyConstraint(['session_id'], [u'observing_session.id'], ),
+                    sa.ForeignKeyConstraint(['session_id'], ['observing_session.id'], ),
                     sa.PrimaryKeyConstraint('obsid')
                     )
 
@@ -65,7 +65,7 @@ def upgrade():
                     sa.Column('size', sa.BigInteger(), nullable=False),
                     sa.Column('md5', sa.String(length=32), nullable=False),
                     sa.Column('source', sa.String(length=64), nullable=False),
-                    sa.ForeignKeyConstraint(['obsid'], [u'observation.obsid'], ),
+                    sa.ForeignKeyConstraint(['obsid'], ['observation.obsid'], ),
                     sa.PrimaryKeyConstraint('name')
                     )
 
@@ -75,16 +75,16 @@ def upgrade():
                     sa.Column('time', sa.DateTime(), nullable=False),
                     sa.Column('type', sa.String(length=64), nullable=True),
                     sa.Column('payload', sa.Text(), nullable=True),
-                    sa.ForeignKeyConstraint(['name'], [u'file.name'], ),
+                    sa.ForeignKeyConstraint(['name'], ['file.name'], ),
                     sa.PrimaryKeyConstraint('id')
                     )
 
     op.create_table('file_instance',
                     sa.Column('store', sa.BigInteger(), nullable=False),
                     sa.Column('parent_dirs', sa.String(length=128), nullable=False),
                     sa.Column('name', sa.String(length=256), nullable=False),
-                    sa.ForeignKeyConstraint(['name'], [u'file.name'], ),
-                    sa.ForeignKeyConstraint(['store'], [u'store.id'], ),
+                    sa.ForeignKeyConstraint(['name'], ['file.name'], ),
+                    sa.ForeignKeyConstraint(['store'], ['store.id'], ),
                     sa.PrimaryKeyConstraint('store', 'parent_dirs', 'name')
                     )
 

hera_librarian/__init__.py

@@ -2,11 +2,11 @@
 # Copyright 2016 the HERA Team.
 # Licensed under the BSD License.
 
-from __future__ import absolute_import, division, print_function
+
 
 import json
 import os.path
-import urllib
+import urllib.request, urllib.parse, urllib.error
 
 __all__ = str('''
 NoSuchConnectionError
@@ -15,7 +15,7 @@
 ''').split()
 
 
-__version__ = "0.1.7a0"
+__version__ = "1.0.0"
 
 
 class NoSuchConnectionError (Exception):
@@ -84,9 +84,9 @@ def _do_http_post(self, operation, **kwargs):
                 kwargs.pop(k)
         req_json = json.dumps(kwargs)
 
-        params = urllib.urlencode({'request': req_json})
+        params = urllib.parse.urlencode({'request': req_json})
         url = self.config['url'] + 'api/' + operation
-        f = urllib.urlopen(url, params)
+        f = urllib.request.urlopen(url, params)
         reply = f.read()
         try:
             reply_json = json.loads(reply)

hera_librarian/base_store.py

@@ -10,7 +10,7 @@
 
 """
 
-from __future__ import absolute_import, division, print_function, unicode_literals
+
 
 __all__ = str('''
 BaseStore
@@ -79,7 +79,7 @@ def _ssh_slurp(self, command, input=None):
 
         if proc.returncode != 0:
             raise RPCError(argv, 'exit code %d; stdout:\n\n%r\n\nstderr:\n\n%r'
-                           % (proc.returncode, stdout, stderr))
+                           % (proc.returncode, stdout.decode("utf-8"), stderr.decode("utf-8")))
 
         return stdout
 
@@ -139,7 +139,7 @@ def copy_to_store(self, local_path, store_path):
         ]
         success = False
 
-        for i in xrange(NUM_RSYNC_TRIES):
+        for i in range(NUM_RSYNC_TRIES):
             proc = subprocess.Popen(argv, shell=False, stdout=subprocess.PIPE,
                                     stderr=subprocess.STDOUT)
             output = proc.communicate()[0]
@@ -235,7 +235,8 @@ def _create_tempdir(self, key='libtmp'):
         path".
 
         """
-        output = self._ssh_slurp('mktemp -d -p %s %s.XXXXXX' % (self.path_prefix, key))
+        # we need to convert the output of _ssh_slurp -- a path in bytes -- to a string
+        output = self._ssh_slurp('mktemp -d -p %s %s.XXXXXX' % (self.path_prefix, key)).decode("utf-8")
         fullpath = output.splitlines()[-1].strip()
 
         if not fullpath.startswith(self.path_prefix):

hera_librarian/cli.py

@@ -6,7 +6,7 @@
 
 """
 
-from __future__ import absolute_import, division, print_function
+
 
 
 import argparse
@@ -91,7 +91,7 @@ def print_table(dict_list, col_list=None, col_names=None):
     for item in dict_list:
         myList.append([str(item[col] or '') for col in col_list])
     # figure out the maximum size for each column
-    colSize = [max(map(len, col)) for col in zip(*myList)]
+    colSize = [max(list(map(len, col))) for col in zip(*myList)]
     formatStr = ' | '.join(["{{:<{}}}".format(i) for i in colSize])
     myList.insert(1, ['-' * i for i in colSize])  # Seperating line
     for item in myList:
@@ -618,7 +618,7 @@ def str_or_huh(x):
     n_deleted = 0
     n_error = 0
 
-    for fname, stats in sorted(allstats.iteritems(), key=lambda t: t[0]):
+    for fname, stats in sorted(iter(allstats.items()), key=lambda t: t[0]):
         nd = stats.get("n_deleted", 0)
         nk = stats.get("n_kept", 0)
         ne = stats.get("n_error", 0)

hera_librarian/tests/test_base_store.py

@@ -40,7 +40,7 @@ def test_path(local_store):
 
 def test_ssh_slurp(local_store):
     # test simple command
-    assert local_store[0]._ssh_slurp("echo hello world") == "hello world\n"
+    assert local_store[0]._ssh_slurp("echo hello world").decode("utf-8") == "hello world\n"
 
     # try a bogus command
     with pytest.raises(RPCError):
@@ -63,7 +63,7 @@ def test_copy_to_store(tmpdir, local_store):
 
     # check that it exists
     dirpath = os.path.join(local_store[1], "test_directory")
-    assert local_store[0]._ssh_slurp("ls {}".format(dirpath)) == "my_file.txt\n"
+    assert local_store[0]._ssh_slurp("ls {}".format(dirpath)).decode("utf-8") == "my_file.txt\n"
 
     # clean up
     shutil.rmtree(os.path.join(local_store[1]))
@@ -79,7 +79,7 @@ def test_chmod(local_store):
     local_store[0]._chmod("my_empty_file", "664")
 
     # make sure permissions are correct
-    output = local_store[0]._ssh_slurp("ls -l {}".format(temppath))
+    output = local_store[0]._ssh_slurp("ls -l {}".format(temppath)).decode("utf-8")
     perms = output.split(" ")[0]
     assert perms == "-rw-rw-r--"
 
@@ -95,7 +95,7 @@ def test_move(local_store):
     local_store[0]._ssh_slurp("touch {}".format(temppath))
     local_store[0]._move("my_empty_file", "my_moved_file")
     temppath2 = os.path.join("/tmp", local_store[1], "my_moved_file")
-    assert local_store[0]._ssh_slurp("ls {}".format(temppath2)) == "{}\n".format(temppath2)
+    assert local_store[0]._ssh_slurp("ls {}".format(temppath2)).decode("utf-8") == "{}\n".format(temppath2)
 
     # test trying to overwrite a file that already exists
     with pytest.raises(RPCError):
@@ -106,7 +106,7 @@ def test_move(local_store):
     local_store[0]._ssh_slurp("rm -f {} {}".format(temppath, temppath2))
     local_store[0]._ssh_slurp("touch {}".format(temppath))
     local_store[0]._move("my_empty_file", "my_moved_file", chmod_spec=664)
-    output = local_store[0]._ssh_slurp("ls -l {}".format(temppath2))
+    output = local_store[0]._ssh_slurp("ls -l {}".format(temppath2)).decode("utf-8")
     perms = output.split(" ")[0]
     assert perms == "-rw-rw-r--"
 
@@ -124,7 +124,7 @@ def test_delete(local_store):
     assert (
         local_store[0]._ssh_slurp(
             "if [ -f {} ]; then echo file_still_exists; fi".format(temppath)
-        )
+        ).decode("utf-8")
         == ""
     )
 
@@ -135,7 +135,7 @@ def test_delete(local_store):
     assert (
         local_store[0]._ssh_slurp(
             "if [ -d {} ]; then echo dir_still_exists; fi".format(tempdir)
-        )
+        ).decode("utf-8")
         == ""
     )
 
@@ -156,7 +156,7 @@ def test_create_tempdir(local_store):
     assert len(tmppath) == len("libtmp.") + 6
     # make sure it exists on the host
     assert (
-        local_store[0]._ssh_slurp("ls -d1 {}/{}".format(tempdir, tmppath))
+        local_store[0]._ssh_slurp("ls -d1 {}/{}".format(tempdir, tmppath)).decode("utf-8")
         == "{}/{}\n".format(tempdir, tmppath)
     )
 

hera_librarian/tests/test_cli.py

@@ -6,7 +6,7 @@
 
 """
 
-from __future__ import print_function, division, absolute_import
+
 import pytest
 import os
 
@@ -131,4 +131,4 @@ def test_generate_parser():
 def test_main(script_runner):
     version = hera_librarian.__version__
     ret = script_runner.run("librarian", "-V")
-    assert ret.stderr == "librarian {}\n".format(version)
+    assert ret.stdout == "librarian {}\n".format(version)

hera_librarian/utils.py

@@ -169,11 +169,11 @@ def all_files():
         locale.setlocale(locale.LC_COLLATE, 'C')
 
         for f in sorted(all_files()):
-            subhash = _md5_of_file(f)
+            subhash = _md5_of_file(f).encode("utf-8")
             md5.update(subhash)  # this is the hex digest, like we want
-            md5.update('  .')  # compat with command-line approach
-            md5.update(f[plen:])
-            md5.update('\n')
+            md5.update('  .'.encode("utf-8"))  # compat with command-line approach
+            md5.update(f[plen:].encode("utf-8"))
+            md5.update('\n'.encode("utf-8"))
     finally:
         locale.setlocale(locale.LC_COLLATE, prevlocale)
 

librarian_server/__init__.py

@@ -7,7 +7,7 @@
 probably ways to work around that but things work well enough as is.
 
 """
-from __future__ import absolute_import, division, print_function, unicode_literals
+
 
 import logging
 import sys
@@ -236,7 +236,7 @@ def maybe_add_stores():
     from .dbutil import SQLAlchemyError
     from .store import Store
 
-    for name, cfg in app.config.get('add-stores', {}).iteritems():
+    for name, cfg in app.config.get('add-stores', {}).items():
         prev = Store.query.filter(Store.name == name).first()
         if prev is None:
             store = Store(name, cfg['path_prefix'], cfg['ssh_host'])

librarian_server/bgtasks.py

@@ -11,7 +11,7 @@
 This system requires Tornado.
 
 """
-from __future__ import absolute_import, division, print_function, unicode_literals
+
 
 __all__ = str('''
 BackgroundTask

librarian_server/dbutil.py

@@ -5,7 +5,7 @@
 """Generic database utilities.
 
 """
-from __future__ import absolute_import, division, print_function, unicode_literals
+
 
 __all__ = str('''
 NotNull