Merge pull request #59 from bbc/dev

Merge dev into master post v2.7.0

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Mediagrains Library Changelog
 
+## 2.7.0
+- Dropped all support for Python2.7
+- Moved python3.6 specific submodules in tree
+- Added `GrainWrapper` class to wrap raw essence in Grains.
+- Added `wrap_video_in_gsf` and `wrap_audio_in_gsf` tools to generate GSF files from raw essence.
+- Added `extract_from_gsf` and `gsf_probe` tools to extract essence and metadata from GSF files.
+- Added MyPy as a dependency
+- Deprecated old asyncio code from v2.6
+- Added Asynchronous GSFEncoding using the standard Encoder in a context-manager type workflow.
+- Added Asynchronous GSFDecoding using the standard Decoder in a context-manager type workflow.
+
 ## 2.6.0
 - Added support for async methods to gsf decoder in python 3.6+
 - Added `Grain.origin_timerange` method.

Jenkinsfile

@@ -1,9 +1,9 @@
 @Library("rd-apmm-groovy-ci-library@v1.x") _
 
 /*
- Runs the following steps in parallel and reports results to GitHub:
+ Runs the following steps in series and reports results to GitHub:
  - Lint using flake8
- - Run Python 2.7 unit tests in tox
+ - Type check using mypy
  - Run Python 3 unit tests in tox
  - Build Debian packages for supported Ubuntu versions
 
@@ -44,84 +44,62 @@ pipeline {
         }
         stage ("Tests") {
             stages {
-                stage ("Py2.7 Linting Check") {
+                stage ("Py36 Linting Check") {
                     steps {
                         script {
-                            env.lint27_result = "FAILURE"
+                            env.lint3_result = "FAILURE"
                         }
-                        bbcGithubNotify(context: "lint/flake8_27", status: "PENDING")
+                        bbcGithubNotify(context: "lint/flake8_3", status: "PENDING")
                         // Run the linter
-                        sh 'python2.7 -m flake8 --filename=mediagrains/*.py,tests/test_*.py'
+                        sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-lint make lint'
                         script {
-                            env.lint27_result = "SUCCESS" // This will only run if the sh above succeeded
+                            env.lint3_result = "SUCCESS" // This will only run if the sh above succeeded
                         }
                     }
                     post {
                         always {
-                            bbcGithubNotify(context: "lint/flake8_27", status: env.lint27_result)
+                            bbcGithubNotify(context: "lint/flake8_3", status: env.lint3_result)
                         }
                     }
                 }
-                stage ("Py36 Linting Check") {
+                stage ("Py36 Type Check") {
                     steps {
                         script {
-                            env.lint3_result = "FAILURE"
+                            env.mypy_result = "FAILURE"
                         }
-                        bbcGithubNotify(context: "lint/flake8_3", status: "PENDING")
+                        bbcGithubNotify(context: "type/mypy", status: "PENDING")
                         // Run the linter
-                        sh 'python3 -m flake8 --filename=mediagrains/*.py,mediagrains_async/*.py,tests/test_*.py,tests/atest_*.py'
+                        sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-mypy make mypy'
                         script {
-                            env.lint3_result = "SUCCESS" // This will only run if the sh above succeeded
+                            env.mypy_result = "SUCCESS" // This will only run if the sh above succeeded
                         }
                     }
                     post {
                         always {
-                            bbcGithubNotify(context: "lint/flake8_3", status: env.lint3_result)
+                            bbcGithubNotify(context: "type/mypy", status: env.mypy_result)
                         }
                     }
                 }
                 stage ("Build Docs") {
-                   steps {
-                       sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-docs make docs'
-                   }
+                    steps {
+                        sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-docs make docs'
+                    }
                 }
-                stage ("Unit Tests") {
-                    stages {
-                        stage ("Python 2.7 Unit Tests") {
-                            steps {
-                                script {
-                                    env.py27_result = "FAILURE"
-                                }
-                                bbcGithubNotify(context: "tests/py27", status: "PENDING")
-                                // Use a workdirectory in /tmp to avoid shebang length limitation
-                                sh 'tox -e py27 --recreate --workdir /tmp/$(basename ${WORKSPACE})/tox-py27'
-                                script {
-                                    env.py27_result = "SUCCESS" // This will only run if the sh above succeeded
-                                }
-                            }
-                            post {
-                                always {
-                                    bbcGithubNotify(context: "tests/py27", status: env.py27_result)
-                                }
-                            }
+                stage ("Python 3 Unit Tests") {
+                    steps {
+                        script {
+                            env.py36_result = "FAILURE"
                         }
-                        stage ("Python 3 Unit Tests") {
-                            steps {
-                                script {
-                                    env.py36_result = "FAILURE"
-                                }
-                                bbcGithubNotify(context: "tests/py36", status: "PENDING")
-                                // Use a workdirectory in /tmp to avoid shebang length limitation
-                                sh 'tox -e py36 --recreate --workdir /tmp/$(basename ${WORKSPACE})/tox-py36'
-                                script {
-                                    env.py36_result = "SUCCESS" // This will only run if the sh above succeeded
-                                }
-                            }
-                            post {
-                                always {
-                                    bbcGithubNotify(context: "tests/py36", status: env.py36_result)
-                                }
-                            }
+                        bbcGithubNotify(context: "tests/py36", status: "PENDING")
+                        // Use a workdirectory in /tmp to avoid shebang length limitation
+                        sh 'TOXDIR=/tmp/$(basename ${WORKSPACE})/tox-py36 make test'
+                        script {
+                            env.py36_result = "SUCCESS" // This will only run if the sh above succeeded
+                        }
+                    }
+                    post {
+                        always {
+                            bbcGithubNotify(context: "tests/py36", status: env.py36_result)
                         }
                     }
                 }
@@ -218,7 +196,6 @@ pipeline {
                         }
                         bbcGithubNotify(context: "pypi/upload", status: "PENDING")
                         sh 'rm -rf dist/*'
-                        bbcMakeGlobalWheel("py27")
                         bbcMakeGlobalWheel("py36")
                         bbcTwineUpload(toxenv: "py36", pypi: true)
                         script {
@@ -246,7 +223,6 @@ pipeline {
                         }
                         bbcGithubNotify(context: "artifactory/upload", status: "PENDING")
                         sh 'rm -rf dist/*'
-                        bbcMakeGlobalWheel("py27")
                         bbcMakeGlobalWheel("py36")
                         bbcTwineUpload(toxenv: "py36", pypi: false)
                         script {
@@ -276,9 +252,9 @@ pipeline {
                         script {
                             for (def dist in bbcGetSupportedUbuntuVersions()) {
                                 bbcDebUpload(sourceFiles: "_result/${dist}-amd64/*",
-                                             removePrefix: "_result/${dist}-amd64",
-                                             dist: "${dist}",
-                                             apt_repo: "ap/python")
+                                                removePrefix: "_result/${dist}-amd64",
+                                                dist: "${dist}",
+                                                apt_repo: "ap/python")
                             }
                         }
                         script {

MANIFEST.in

@@ -3,6 +3,5 @@ include tox.ini
 include COPYING
 recursive-include examples *.gsf
 recursive-include tests *.py
-recursive-include mediagrains_py36 *.py
 include ICLA.md
 include LICENSE.md

Makefile

@@ -1,9 +1,7 @@
-PYTHON=`which python`
-PYTHON2=`which python2`
 PYTHON3=`which python3`
 PY2DSC=`which py2dsc`
 
-PY2DSC_PARAMS?=--with-python2=true --with-python3=true
+PY2DSC_PARAMS?=--with-python2=false --with-python3=true
 
 topdir := $(realpath $(dir $(lastword $(MAKEFILE_LIST))))
 topbuilddir := $(realpath .)
@@ -15,7 +13,7 @@ MODNAME=$(PROJECT)
 
 # The rules for names and versions in python, rpm, and deb are different
 # and not entirely compatible. As such py2dsc will automatically convert
-# your package name into a suitable deb name and version number, and this 
+# your package name into a suitable deb name and version number, and this
 # code replicates that.
 DEBNAME=$(shell echo $(MODNAME) | tr '[:upper:]_' '[:lower:]-')
 DEBVERSION=$(shell echo $(VERSION) | sed 's/\.dev/~dev/')
@@ -49,15 +47,15 @@ $(topbuilddir)/dist:
 	mkdir -p $@
 
 source: $(topbuilddir)/dist
-	$(PYTHON) $(topdir)/setup.py sdist $(COMPILE) --dist-dir=$(topbuilddir)/dist
+	$(PYTHON3) $(topdir)/setup.py sdist $(COMPILE) --dist-dir=$(topbuilddir)/dist
 
 $(topbuilddir)/dist/$(MODNAME)-$(VERSION).tar.gz: source
 
 install:
-	$(PYTHON) $(topdir)/setup.py install --root $(DESTDIR) $(COMPILE)
+	$(PYTHON3) $(topdir)/setup.py install --root $(DESTDIR) $(COMPILE)
 
 clean:
-	$(PYTHON) $(topdir)/setup.py clean || true
+	$(PYTHON3) $(topdir)/setup.py clean || true
 	rm -rf $(topbuilddir)/.tox
 	rm -rf $(topbuilddir)/build/ MANIFEST
 	rm -rf $(topbuilddir)/dist
@@ -67,13 +65,10 @@ clean:
 	find $(topdir) -name '*.py,cover' -delete
 	rm -rf $(topbuilddir)/docs
 
-testenv: $(TOXDIR)/py27/bin/activate $(TOXDIR)/py3/bin/activate
+testenv: $(TOXDIR)/py36/bin/activate
 
-$(TOXDIR)/py3/bin/activate: tox.ini
-	tox -e py3 --recreate --workdir $(TOXDIR)
-
-$(TOXDIR)/py27/bin/activate: tox.ini
-	tox -e py27 --recreate --workdir $(TOXDIR)
+$(TOXDIR)/py36/bin/activate: tox.ini
+	tox -e py36 --recreate --notest --workdir $(TOXDIR)
 
 test:
 	tox --workdir $(TOXDIR)
@@ -117,11 +112,9 @@ rpm: $(RPM_PREFIX)/SPECS/$(MODNAME).spec $(RPM_PREFIX)/SOURCES/$(MODNAME)-$(VERS
 	cp $(RPM_PREFIX)/RPMS/*/*.rpm $(topbuilddir)/dist
 
 wheel:
-	$(PYTHON2) $(topdir)/setup.py bdist_wheel
 	$(PYTHON3) $(topdir)/setup.py bdist_wheel
 
 egg:
-	$(PYTHON2) $(topdir)/setup.py bdist_egg
 	$(PYTHON3) $(topdir)/setup.py bdist_egg
 
 docs: $(topbuilddir)/docs/$(MODNAME).html
@@ -130,7 +123,13 @@ $(topbuilddir)/docs/$(MODNAME):
 	mkdir -p $(topbuilddir)/docs
 	ln -s $(topdir)/$(MODNAME) $(topbuilddir)/docs/
 
-$(topbuilddir)/docs/$(MODNAME).html: $(topbuilddir)/docs/$(MODNAME) $(TOXDIR)/py3/bin/activate
-	. $(TOXDIR)/py3/bin/activate && cd $(topbuilddir)/docs/ && pydoc -w ./
+$(topbuilddir)/docs/$(MODNAME).html: $(topbuilddir)/docs/$(MODNAME) $(TOXDIR)/py36/bin/activate
+	. $(TOXDIR)/py36/bin/activate && cd $(topbuilddir)/docs/ && pydoc -w ./
+
+lint: $(TOXDIR)/py36/bin/activate
+	. $(TOXDIR)/py36/bin/activate && python -m flake8
+
+mypy: $(TOXDIR)/py36/bin/activate
+	. $(TOXDIR)/py36/bin/activate && python -m mypy -p $(MODNAME)
 
-.PHONY: test testenv clean install source deb dsc rpm wheel egg all rpm_dirs rpm_spec docs
+.PHONY: test testenv clean install source deb dsc rpm wheel egg all rpm_dirs rpm_spec docs lint mypy

README.md

@@ -10,11 +10,14 @@ that nicely wrap those grains, as well as a full serialisation and
 deserialisation library for GSF format. Please read the pydoc
 documentation for more details.
 
+Some useful tools for handling the Grain Sequence Format (GSF) file format
+are also included - see [Tools](#tools).
+
 ## Installation
 
 ### Requirements
 
-* A working Python 2.7 or Python 3.6+ installation
+* A working Python 3.6+ installation
 * BBC R&D's internal deb repository set up as a source for apt (if installing via apt-get)
 * The tool [tox](https://tox.readthedocs.io/en/latest/) is needed to run the unittests, but not required to use the library.
 
@@ -25,7 +28,7 @@ documentation for more details.
 $ pip install mediagrains
 
 # Install via apt-get
-$ apt-get install python-mediagrains python3-mediagrains
+$ apt-get install python3-mediagrains
 
 # Install directly from source repo
 $ git clone git@github.com:bbc/rd-apmm-python-lib-mediagrains.git
@@ -85,7 +88,7 @@ it with colour-bars:
 ...             i += 1
 ```
 
-(In python3.6+ a more natural interface for accessing data exists in the form of numpy arrays. See later.)
+(a more natural interface for accessing data exists in the form of numpy arrays. See later.)
 
 The object grain can then be freely used for whatever video processing
 is desired, or it can be serialised into a GSF file as follows:
@@ -161,9 +164,9 @@ between two grains, both as a printed string (as seen above) and also
 in a data-centric fashion as a tree structure which can be
 interrogated in code.
 
-### Numpy arrays (Python 3.6+)
+### Numpy arrays
 
-In python 3.6 or higher an additional feature is provided in the form of numpy array access to the data in a grain. As such the above example of creating colourbars can be done more easily:
+An additional feature is provided in the form of numpy array access to the data in a grain. As such the above example of creating colourbars can be done more easily:
 
 ```Python console
 >>> from mediagrains.numpy import VideoGrain
@@ -193,6 +196,30 @@ The API is well documented in the docstrings of the module mediagrains, to view:
 pydoc mediagrains
 ```
 
+## Tools
+Some tools are installed with the library to make working with the Grain Sequence Format (GSF) file format easier.
+
+* `wrap_video_in_gsf` - Provides a means to read raw video essence and generate a GSF file.
+* `wrap_audio_in_gsf` - As above, but for audio.
+* `extract_from_gsf` - Read a GSF file and dump out the raw essence within.
+* `gsf_probe` - Read metadata about the segments in a GSF file.
+
+For example, to generate a GSF file containing a test pattern from `ffmpeg`, dump the metadata and then play it out
+again:
+```bash
+ffmpeg -f lavfi -i testsrc=duration=20:size=1920x1080:rate=25 -pix_fmt yuv422p10le -c:v rawvideo -f rawvideo - | \
+wrap_video_in_gsf - output.gsf --size 1920x1080 --format S16_422_10BIT --rate 25
+gsf_probe output.gsf
+extract_gsf_essence output.gsf - | ffplay -f rawvideo -pixel_format yuv422p10 -video_size 1920x1080 -framerate 25 pipe:0
+```
+
+To do the same with a sine wave:
+```bash
+ffmpeg -f lavfi -i "sine=frequency=1000:duration=5" -f s16le -ac 2 - | wrap_audio_in_gsf - output_audio.gsf --sample-rate 44100
+gsf_probe output_audio.gsf
+extract_gsf_essence output_audio.gsf - | ffplay -f s16le -ac 2 -ar 44100 pipe:0
+```
+
 ## Development
 ### Testing
 

mediagrains/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/python
 #
 # Copyright 2018 British Broadcasting Corporation
 #
@@ -40,7 +39,6 @@
 The gsf and grain submodules have their own documentation.
 """
 
-from __future__ import absolute_import
 from .grain_constructors import Grain, VideoGrain, CodedVideoGrain, AudioGrain, CodedAudioGrain, EventGrain
 
 __all__ = ["Grain", "VideoGrain", "CodedVideoGrain", "AudioGrain", "CodedAudioGrain", "EventGrain"]