Sparse/fallocate/uid/gid options for whisper-resize.py (#332)

* Sparse/fallocate/uid/gid options for whisper-resize.py

* Python 2.7 is not available anymore

* Adding Python 3.12

* Taming linter

* Excluding contrib from tests

.github/workflows/tests.yml

@@ -13,9 +13,6 @@ jobs:
     strategy:
       matrix:
         include:
-          - name: "Python 2.7"
-            python-version: '2.7'
-            toxenv: "py27"
           - name: "Pypy 2.7"
             python-version: 'pypy-2.7'
             toxenv: 'pypy'
@@ -37,6 +34,9 @@ jobs:
           - name: "Python 3.11"
             python-version: '3.11'
             toxenv: 'py3'
+          - name: "Python 3.12"
+            python-version: '3.12'
+            toxenv: 'py3'
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
@@ -54,19 +54,19 @@ jobs:
         run: |
           tox
       - name: Linting
-        if: ${{ matrix.python-version==3.11 }}
+        if: ${{ matrix.python-version==3.12 }}
         env:
           TOXENV: lint
         run: |
           tox
       - name: Run Codecov
-        if: ${{ matrix.python-version==3.11 }}
+        if: ${{ matrix.python-version==3.12 }}
         env:
           TOXENV: lint
         run: |
           pip install codecov
           codecov
       - name: Upload coverage to Codecov
-        if: ${{ matrix.python-version==3.11 }}
+        if: ${{ matrix.python-version==3.12 }}
         uses: codecov/codecov-action@v3
 

bin/whisper-resize.py

@@ -42,13 +42,31 @@
 option_parser.add_option(
     '--newfile', default=None, action='store',
     help="Create a new database file without removing the existing one")
+option_parser.add_option(
+    '--sparse', default=True, action='store_true',
+    help="Create new whisper as sparse file")
+option_parser.add_option(
+    '--fallocate', default=False, action='store_true',
+    help="Create new whisper and use fallocate (disabling sparse)")
+option_parser.add_option(
+    '--chown-uid', default=-1,
+    type='int', help="Run chown for specific UID")
+option_parser.add_option(
+    '--chown-gid', default=-1,
+    type='int', help="Run chown for specific GID")
 option_parser.add_option(
     '--nobackup', action='store_true',
     help='Delete the .bak file after successful execution')
 option_parser.add_option(
     '--aggregate', action='store_true',
     help='Try to aggregate the values to fit the new archive better.'
          ' Note that this will make things slower and use more memory.')
+option_parser.add_option(
+    '--quiet', default=False, action='store_true',
+    help='Print less messages')
+option_parser.add_option(
+    '--silent', default=False, action='store_true',
+    help='Print no messages')
 
 (options, args) = option_parser.parse_args()
 
@@ -63,6 +81,12 @@
   option_parser.print_help()
   sys.exit(1)
 
+if not options.silent:
+    size = os.stat(path).st_size
+    blocks = os.stat(path).st_blocks
+    print('Old file: %s (%d bytes, %d blocks*%d=%d bytes on disk)'
+          % (path, size, blocks, 512, blocks * 512))
+
 info = whisper.info(path)
 
 new_archives = [whisper.parseRetentionDef(retentionDef)
@@ -82,7 +106,8 @@
 else:
   aggregationMethod = options.aggregationMethod
 
-print('Retrieving all data from the archives')
+if not options.quiet and not options.silent:
+    print('Retrieving all data from the archives')
 for archive in old_archives:
   fromTime = now - archive['retention'] + archive['secondsPerPoint']
   untilTime = now
@@ -92,20 +117,33 @@
 if options.newfile is None:
   tmpfile = path + '.tmp'
   if os.path.exists(tmpfile):
-    print('Removing previous temporary database file: %s' % tmpfile)
+    if not options.quiet and not options.silent:
+        print('Removing previous temporary database file: %s' % tmpfile)
     os.unlink(tmpfile)
   newfile = tmpfile
 else:
   newfile = options.newfile
 
-print('Creating new whisper database: %s' % newfile)
-whisper.create(newfile, new_archives, xFilesFactor=xff, aggregationMethod=aggregationMethod)
+if not options.quiet and not options.silent:
+    print('Creating new whisper database: %s' % newfile)
+
+try:
+    whisper.create(newfile, new_archives, xFilesFactor=xff,
+                   aggregationMethod=aggregationMethod, sparse=options.sparse,
+                   useFallocate=options.fallocate)
+except whisper.WhisperException as exc:
+    raise SystemExit('[ERROR] %s' % str(exc))
+
 size = os.stat(newfile).st_size
-print('Created: %s (%d bytes)' % (newfile, size))
+blocks = os.stat(newfile).st_blocks
+if not options.quiet and not options.silent:
+    print('Created: %s (%d bytes, %d blocks*%d=%d bytes on disk)'
+          % (newfile, size, blocks, 512, blocks * 512))
 
 if options.aggregate:
   # This is where data will be interpolated (best effort)
-  print('Migrating data with aggregation...')
+  if not options.quiet and not options.silent:
+    print('Migrating data with aggregation...')
   all_datapoints = []
   for archive in sorted(old_archives, key=lambda x: x['secondsPerPoint']):
     # Loading all datapoints into memory for fast querying
@@ -126,8 +164,8 @@
 
   oldtimestamps = list(map(lambda p: p[0], all_datapoints))
   oldvalues = list(map(lambda p: p[1], all_datapoints))
-
-  print("oldtimestamps: %s" % oldtimestamps)
+  if not options.quiet and not options.silent:
+    print("oldtimestamps: %s" % oldtimestamps)
   # Simply cleaning up some used memory
   del all_datapoints
 
@@ -138,9 +176,11 @@
     step = archive['secondsPerPoint']
     fromTime = now - archive['retention'] + now % step
     untilTime = now + now % step + step
-    print("(%s,%s,%s)" % (fromTime, untilTime, step))
+    if not options.quiet and not options.silent:
+      print("(%s,%s,%s)" % (fromTime, untilTime, step))
     timepoints_to_update = range(fromTime, untilTime, step)
-    print("timepoints_to_update: %s" % timepoints_to_update)
+    if not options.quiet and not options.silent:
+      print("timepoints_to_update: %s" % timepoints_to_update)
     newdatapoints = []
     for tinterval in zip(timepoints_to_update[:-1], timepoints_to_update[1:]):
       # TODO: Setting lo= parameter for 'lefti' based on righti from previous
@@ -157,7 +197,8 @@
                                                   non_none, newvalues)])
     whisper.update_many(newfile, newdatapoints)
 else:
-  print('Migrating data without aggregation...')
+  if not options.quiet and not options.silent:
+    print('Migrating data without aggregation...')
   for archive in old_archives:
     timeinfo, values = archive['data']
     datapoints = zip(range(*timeinfo), values)
@@ -168,18 +209,34 @@
   sys.exit(0)
 
 backup = path + '.bak'
-print('Renaming old database to: %s' % backup)
+if not options.quiet and not options.silent:
+    print('Renaming old database to: %s' % backup)
 os.rename(path, backup)
 
 try:
-  print('Renaming new database to: %s' % path)
+  if not options.quiet and not options.silent:
+    print('Renaming new database to: %s' % path)
   os.rename(tmpfile, path)
 except (OSError):
   traceback.print_exc()
-  print('\nOperation failed, restoring backup')
+  if not options.quiet and not options.silent:
+    print('\nOperation failed, restoring backup')
   os.rename(backup, path)
   sys.exit(1)
 
+if options.chown_uid > 0 and options.chown_gid > 0:
+  try:
+    os.chown(path=path, uid=options.chown_uid, gid=options.chown_gid)
+  except (OSError):
+    traceback.print_exc()
+
+if not options.silent:
+    size = os.stat(path).st_size
+    blocks = os.stat(path).st_blocks
+    print('New file: %s (%d bytes, %d blocks*%d=%d bytes on disk)'
+          % (path, size, blocks, 512, blocks * 512))
+
 if options.nobackup:
-  print("Unlinking backup: %s" % backup)
+  if not options.quiet and not options.silent:
+    print("Unlinking backup: %s" % backup)
   os.unlink(backup)

tox.ini

@@ -9,6 +9,7 @@ envlist =
   py39,
   py310,
   py311,
+  py312,
   pypy,
   pypy3,
   lint,
@@ -25,8 +26,6 @@ commands =
   sh -c 'bin/whisper-resize.py --help >/dev/null'
   sh -c 'bin/whisper-set-aggregation-method.py --help >/dev/null'
   sh -c 'bin/whisper-update.py --help >/dev/null'
-  sh -c 'contrib/whisper-auto-update.py --help >/dev/null'
-  sh -c 'contrib/whisper-auto-resize.py --help >/dev/null'
   coverage run --branch --include='whisper.py,test_whisper.py, bin/whisper-resize.py' test_whisper.py
   coverage xml
   coverage report