Skip to content

Commit 622b8ff

Browse files
sherifersonsheriferson
authored
Merge pull request #559 from sheriferson/feature/github-actions
Python 3.9/3.11 support, dropping Python 3.6 support, and moving to GitHub Actions
2 parents 2599bc1 + 9643dc1 commit 622b8ff

31 files changed

+237
-129
lines changed

‎.bumpversion.cfg

Lines changed: 0 additions & 20 deletions
This file was deleted.

‎.circleci/config.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ jobs:
4949
name: install test dependencies
5050
command: |
5151
. venv/bin/activate
52-
pip install -r test_requirements.txt
52+
pip install -r requirements-test.txt
5353
5454
# run tests!
5555
- run:

‎.github/workflows/github-actions-ci.yml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: ci/github-actions
2+
3+
on:
4+
pull_request:
5+
branches: [ master, feature/** ]
6+
7+
push:
8+
branches: [ master, feature/** ]
9+
10+
11+
jobs:
12+
run-light-tests:
13+
env:
14+
SKIP_LIMS: true
15+
TEST_INHOUSE: false
16+
ALLOW_TEST_DOWNLOADS: false
17+
TEST_API_ENDPOINT: "http://api.brain-map.org"
18+
IPFX_TEST_TIMEOUT: 60
19+
20+
runs-on: ubuntu-latest
21+
strategy:
22+
matrix:
23+
python-version: ["3.9", "3.11"]
24+
25+
steps:
26+
- uses: actions/checkout@v3
27+
- name: Set up Python ${{ matrix.python-version }}
28+
uses: actions/setup-python@v4
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
- name: Install dependencies
32+
run: |
33+
sudo apt-get update
34+
sudo apt-get install -y hdf5-tools curl
35+
python -m pip install --upgrade pip
36+
pip install -r requirements.txt
37+
pip install allensdk
38+
- name: Run tests
39+
run: |
40+
pip install -r requirements-test.txt
41+
mkdir -p test-results
42+
git lfs install
43+
git config lfs.url 'https://github.com/AllenInstitute/ipfx.git/info/lfs'
44+
git lfs env
45+
git lfs pull
46+
python -m pytest --junitxml=test-results/junit.xml --verbose
47+
48+
49+
onprem-tests:
50+
name: on-prem tests
51+
runs-on: ["self-hosted"]
52+
strategy:
53+
matrix:
54+
image: ["ipfx_py39:latest", "ipfx_py311:latest"]
55+
steps:
56+
- uses: actions/checkout@v4
57+
- name: run test in docker
58+
run: |
59+
docker run \
60+
--env-file ~/env.list \
61+
--mount type=bind,source=$PWD,target=/local1/github_worker,bind-propagation=rshared \
62+
-v /data/informatics/module_test_data/:/data/informatics/module_test_data/ \
63+
-v /allen/aibs/informatics/module_test_data/:/allen/aibs/informatics/module_test_data/ \
64+
-v /allen/programs/celltypes/production/mousecelltypes/:/allen/programs/celltypes/production/mousecelltypes/ \
65+
-v /allen/programs/celltypes/workgroups/279/:/allen/programs/celltypes/workgroups/279/ \
66+
-v /allen/programs/celltypes/production/humancelltypes/:/allen/programs/celltypes/production/humancelltypes/ \
67+
--workdir /local1/github_worker --rm \
68+
--user 1001:1001 \
69+
${{ matrix.image }} \
70+
/bin/bash -c "python -m venv .venv; \
71+
source .venv/bin/activate; \
72+
pip install --upgrade pip; \
73+
pip install numpy; \
74+
pip install -r requirements.txt; \
75+
export TEST_API_ENDPOINT=http://api.brain-map.org; \
76+
export SKIP_LIMS=false; \
77+
export TEST_INHOUSE=true; \
78+
export ALLOW_TEST_DOWNLOADS=true; \
79+
export IPFX_TEST_TIMEOUT=60; \
80+
pip install -r requirements-test.txt; \
81+
git config lfs.url 'https://github.com/AllenInstitute/ipfx.git/info/lfs'; \
82+
git lfs env; \
83+
git lfs pull; \
84+
python -m pytest \
85+
--junitxml=test-reports/test.xml --verbose"

‎.github/workflows/nightly-onprem.yml

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: IPFX Nightly onprem tests
2+
on:
3+
schedule:
4+
- cron: '05 6 * * *'
5+
6+
7+
jobs:
8+
onprem:
9+
name: on-prem tests
10+
runs-on: ["self-hosted"]
11+
strategy:
12+
matrix:
13+
image: ["ipfx_py39:latest", "ipfx_py311:latest"]
14+
branch: ["master", "feature/**"]
15+
steps:
16+
- uses: actions/checkout@v4
17+
with:
18+
ref: ${{ matrix.branch }}
19+
- name: run test in docker
20+
run: |
21+
docker run \
22+
--mount type=bind,source=$PWD,target=/local1/github_worker,bind-propagation=rshared \
23+
-v /data/informatics/module_test_data/:/data/informatics/module_test_data/ \
24+
-v /allen/aibs/informatics/module_test_data/:/allen/aibs/informatics/module_test_data/ \
25+
-v /allen/programs/celltypes/production/mousecelltypes/:/allen/programs/celltypes/production/mousecelltypes/ \
26+
-v /allen/programs/celltypes/workgroups/279/:/allen/programs/celltypes/workgroups/279/ \
27+
-v /allen/programs/celltypes/production/humancelltypes/:/allen/programs/celltypes/production/humancelltypes/ \
28+
--workdir /local1/github_worker --rm \
29+
--user 1001:1001 \
30+
${{ matrix.image }} \
31+
/bin/bash -c "python -m venv .venv; \
32+
source .venv/bin/activate; \
33+
pip install --upgrade pip; \
34+
pip install numpy; \
35+
pip install -r requirements.txt; \
36+
export TEST_API_ENDPOINT=http://api.brain-map.org; \
37+
pip install -r requirements-test.txt; \
38+
git config lfs.url 'https://github.com/AllenInstitute/ipfx.git/info/lfs'; \
39+
git lfs env; \
40+
git lfs pull; \
41+
python -m pytest \
42+
--junitxml=test-reports/test.xml --verbose"

‎.gitignore

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,5 +78,14 @@ sandbox/
7878
#linters
7979
.mypy_cache
8080

81-
#virtualenv
82-
venv
81+
# virtualenv / environments
82+
.env
83+
.venv
84+
env/
85+
venv/
86+
ENV/
87+
env.bak/
88+
venv.bak/
89+
90+
# MacOS Finder
91+
.DS_Store

‎AUTHORS.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,4 @@ Credits
1212
* Nile Graddis @nilegraddis
1313
* Sergey Gratiy @sgratiy
1414
* Yang Yu @gnayuy
15+
* Sherif Soliman @sheriferson

‎CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ All notable changes to this project will be documented in this file.
66
### Added
77

88
### Changed
9+
- Removed Python 3.6 support
10+
- Updated dependencies and library for Python 3.9 to 3.11 support
11+
- Moved CI and testing to GitHub Actions
912

1013
## [1.0.8] = 2023-06-29
1114
Changed:

‎CONTRIBUTING.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ visual git plugins, we prefer the following convention for branch naming:
9696
GH-712/bugfix/auto-reward-key
9797
GH-9999/feature/parallel-behavior-analysis
9898
```
99-
* Create an environment and install necessary requirements: `requirements.txt` and `test_requirements.txt`
99+
* Create an environment and install necessary requirements: `requirements.txt` and `requirements-test.txt`
100100
* Start writing code!
101101

102102
### Style guidelines

‎MANIFEST.in

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
include requirements.txt
22
include requirements-test.txt
3-
include requirements-dev.txt
43
include AUTHORS.rst
54
include CHANGELOG.md
65
include README.md

‎README.md

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ We welcome contributions! Please see our [contribution guide](https://github.com
2222

2323
Deprecation Warning
2424
-------------------
25-
The 1.0.0 release of ipfx brings some new features, like NWB2 support, along with improvements to our documentation and testing. We will also drop support for
26-
- NWB1
27-
- Python 2
2825

29-
Older versions of ipfx will continue to be available, but may receive only occasional bugfixes and patches.
26+
The 2.0.0 release of IPFX drops support for Python 3.6 which reached end of life and stopped receiving security updated on December 23, 2021.
27+
IPFX is now tested on Python 3.9 and higher.

‎docker/README.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Docker images for on premise testing
2+
3+
This directory contains Dockerfiles for building images for running on-prem tests that require internal Allen Institute resources. On-prem tests use GitHub self-hosted runners that will run tests on docker images built from these Dockerfiles.
4+
5+
Our light and on-prem tests are defined in [our workflow file](../.github/workflows/github-actions-ci.yml "Link to GitHub Actions workflow for light and on-prem tests").
6+
7+
- See [here](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) for more information on self-hosted runners.
8+
- See [here](https://docs.github.com/en/actions/hosting-your-own-runners/adding-self-hosted-runners) for more information on adding self-hosted runners to a GitHub repository.
9+
10+
## Building images
11+
12+
If you are an Allen Institute developer, you will have instructions on how to access the machine running the IPFX self-hosted runner.
13+
14+
On this machine you can create the Docker image corresponding to the Python versions we test on by running:
15+
16+
```
17+
cd ipfx/docker/py39
18+
docker build -t ipfx_py39:latest .
19+
```
20+
21+
And this should be sufficient for the on-prem tests defined in our GitHub workflow to run.

‎docker/py311/Dockerfile

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
FROM python:3.11
2+
3+
RUN apt-get update \
4+
&& apt-get install -y \
5+
hdf5-tools \
6+
curl \
7+
git-lfs \
8+
&& rm -rf /var/lib/apt/lists/*
9+
10+
RUN curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash

‎docker/py36/Dockerfile renamed to ‎docker/py39/Dockerfile

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
FROM python:3.6
1+
FROM python:3.9
22

33
RUN apt-get update \
44
&& apt-get install -y \
55
hdf5-tools \
6-
curl
6+
curl \
7+
git-lfs \
8+
&& rm -rf /var/lib/apt/lists/*
79

8-
RUN curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash
9-
10-
RUN apt-get install git-lfs
10+
RUN curl https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash

‎ipfx/attach_metadata/sink/metadata_sink.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ def _ensure_plural_targets(
112112
targets = self.targets
113113
elif isinstance(targets, dict):
114114
targets = [targets]
115-
elif isinstance(targets, collections.Sequence):
115+
elif isinstance(targets, collections.abc.Sequence):
116116
targets = list(targets)
117117
else:
118118
raise ValueError(

‎ipfx/attach_metadata/sink/nwb2_sink.py

Lines changed: 0 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,6 @@ def _commit_nwb_changes(self):
9292
recorded until _reload_nwbfile
9393
"""
9494

95-
set_container_sources(self.nwbfile, self._h5_file.filename)
9695
self.nwbfile.set_modified(True)
9796
# Because the NWB schema versions of NWB data produced by MIES are older
9897
# we do not want to cache the newer schema versions that IPFX is currently using
@@ -236,36 +235,3 @@ def serialize(self, targets: Optional[OneOrMany[Dict[str, Any]]] = None):
236235
file_.write(self._data.getvalue())
237236

238237
self._reload_nwbfile()
239-
240-
241-
def set_container_sources(
242-
container: hdmf.container.AbstractContainer,
243-
source: str
244-
):
245-
"""Traverse an NWBFile starting at a given container, setting the
246-
container_source attribute inplace on each container.
247-
248-
Parameters
249-
----------
250-
container : container_source will be set on this object as well as on
251-
each of its applicable children.
252-
source : The new value of container source
253-
"""
254-
children = [container]
255-
while children:
256-
current = children.pop()
257-
258-
# 💀💀💀
259-
# container_source is set on write, but cannot be overrwritten, making
260-
# read -> modify -> write elsewhere
261-
# pretty tricky!
262-
# this is a fragile workaround
263-
if hasattr(current, "_AbstractContainer__container_source"):
264-
setattr(
265-
current,
266-
"_AbstractContainer__container_source",
267-
source
268-
)
269-
270-
if hasattr(current, "children"):
271-
children.extend(current.children)

‎ipfx/dataset/create.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,12 @@ def is_file_mies(path: str) -> bool:
3131
with h5py.File(path, "r") as fil:
3232
if "generated_by" in fil["general"].keys():
3333
generated_by = dict(fil["general"]["generated_by"][:])
34-
return generated_by.get("Package", "None") == "MIES"
34+
try:
35+
decoded_generation_info = {k.decode(): v.decode() for k, v in generated_by.items()}
36+
except:
37+
decoded_generation_info = generated_by
38+
39+
return decoded_generation_info.get("Package", "None") == "MIES"
3540

3641
return False
3742

‎ipfx/version.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
1.0.8
1+
2.0.0

‎ipfx/x_to_nwb/ABFConverter.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,10 +79,10 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T
7979
nwbFile.add_ic_electrode(electrodes)
8080

8181
for i in self._createStimulusSeries(electrodes):
82-
nwbFile.add_stimulus(i)
82+
nwbFile.add_stimulus(i, use_sweep_table=True)
8383

8484
for i in self._createAcquiredSeries(electrodes):
85-
nwbFile.add_acquisition(i)
85+
nwbFile.add_acquisition(i, use_sweep_table=True)
8686

8787
with NWBHDF5IO(outFile, "w") as io:
8888
io.write(nwbFile, cache_spec=True)

‎ipfx/x_to_nwb/DatConverter.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,10 @@ def generateList(multipleGroupsPerFile, pul):
7171
nwbFile.add_ic_electrode(electrodes)
7272

7373
for i in self._createAcquiredSeries(electrodes, elem):
74-
nwbFile.add_acquisition(i)
74+
nwbFile.add_acquisition(i, use_sweep_table=True)
7575

7676
for i in self._createStimulusSeries(electrodes, elem):
77-
nwbFile.add_stimulus(i)
77+
nwbFile.add_stimulus(i, use_sweep_table=True)
7878

7979
if multipleGroupsPerFile:
8080
outFileFmt = outFile

‎test_requirements.txt renamed to ‎requirements-test.txt

File renamed without changes.

‎requirements.txt

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
1-
argschema<2.0.0
21
allensdk
2+
argschema
33
dictdiffer
4-
h5py==2.10.0
5-
marshmallow==3.0.0rc6
6-
matplotlib>=1.4.3
4+
h5py
5+
marshmallow
6+
matplotlib
77
methodtools
8-
numpy>=1.15.4,<1.19.0
9-
pandas>=0.25.1,<=0.25.3
8+
numpy
9+
pandas
1010
pg8000
1111
pillow
12-
pyabf<2.3.0
13-
pynwb>=1.3.2,<2.0.0
14-
pyYAML<6.0.0
15-
scipy>=0.15.1
16-
simplejson>=3.10.0
12+
pyabf
13+
pynwb==2.2.0
14+
pyYAML
15+
ruamel.yaml<0.18.0
16+
scipy
17+
simplejson
1718
watchdog

0 commit comments

Comments
 (0)