Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Performance improvements and collect test improvements #107

Merged
merged 11 commits into from
Sep 11, 2024
Merged

Performance improvements and collect test improvements #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
77c8946
Make squashing faster
dschwoerer Nov 13, 2023
2e48c55
Reduce amount of combinations we test
dschwoerer Nov 13, 2023
ed7c213
Do not force --cov for pytest
dschwoerer Nov 13, 2023
5047746
Remove last reference of gc
dschwoerer Nov 13, 2023
cdeb1d2
Restore parallel squash collect tests
ZedThree Jul 24, 2024
aa67659
Cache dump file creation for collect tests
ZedThree Jul 31, 2024
723e32b
Cut down on collect slicing tests parameterisation
ZedThree Jul 31, 2024
b280a27
Cache dump file creation for collect test with guard cell sizes
ZedThree Jul 31, 2024
4c1f384
Only make one of each kind of field in test dump files
ZedThree Jul 31, 2024
55002e2
Replace duplicated collect tests with parameterised test
ZedThree Jul 31, 2024
f1bd179
Name squash/collect parameters for collect tests
ZedThree Jul 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions pyproject.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ dynamic = ["version"]
[project.optional-dependencies]
tests = [
"pytest",
"pytest-cov",
]
docs = [
"sphinx>=3.4,<5",
Expand All @@ -57,6 +56,3 @@ version = { attr = "setuptools_scm.get_version" }

[tool.setuptools_scm]
write_to = "src/boutdata/_version.py"

[tool.pytest.ini_options]
addopts = "--cov=boutdata"
3 changes: 0 additions & 3 deletions src/boutdata/squashoutput.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ def squashoutput(
is used.
"""
# use local imports to allow fast import for tab-completion
import gc
import glob
import os
import shutil
Expand Down Expand Up @@ -240,7 +239,6 @@ def squashoutput(
f.write(varname, var)

var = None
gc.collect()

# Copy file attributes
for attrname in outputs.list_file_attributes():
Expand All @@ -254,7 +252,6 @@ def squashoutput(
f.close()

del outputs
gc.collect()

if delete:
if append:
Expand Down
62 changes: 11 additions & 51 deletions src/boutdata/tests/make_test_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@
import numpy as np
from netCDF4 import Dataset

field3d_t_list = ["field3d_t_1", "field3d_t_2"]
field3d_list = ["field3d_1", "field3d_2"]
field2d_t_list = ["field2d_t_1", "field2d_t_2"]
field2d_list = ["field2d_1", "field2d_2"]
fieldperp_t_list = ["fieldperp_t_1", "fieldperp_t_2"]
fieldperp_list = ["fieldperp_1", "fieldperp_2"]
scalar_t_list = ["t_array", "scalar_t_1", "scalar_t_2"]
field3d_t_list = ["field3d_t_1"]
field3d_list = ["field3d_1"]
field2d_t_list = ["field2d_t_1"]
field2d_list = ["field2d_1"]
fieldperp_t_list = ["fieldperp_t_1"]
fieldperp_list = ["fieldperp_1"]
scalar_t_list = ["t_array", "scalar_t_1"]

# Note "yindex_global" attribute not included here for FieldPerps, because it is handled
# specially
Expand All @@ -19,61 +19,31 @@
"direction_y": "Standard",
"direction_z": "Standard",
},
"field3d_t_2": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Standard",
},
"field3d_1": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Standard",
},
"field3d_2": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Standard",
},
"field2d_t_1": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Average",
},
"field2d_t_2": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Average",
},
"field2d_1": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Average",
},
"field2d_2": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Average",
},
"fieldperp_t_1": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Standard",
},
"fieldperp_t_2": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Standard",
},
"fieldperp_1": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Standard",
},
"fieldperp_2": {
"cell_location": "CELL_CENTRE",
"direction_y": "Standard",
"direction_z": "Standard",
},
}

expected_file_attributes = {
Expand Down Expand Up @@ -228,7 +198,6 @@ def create3D_t(name):
result[name] = data[:, xslice, yslice, zslice]

create3D_t("field3d_t_1")
create3D_t("field3d_t_2")

def create3D(name):
var = outputfile.createVariable(name, float, ("x", "y", "z"))
Expand All @@ -241,7 +210,6 @@ def create3D(name):
result[name] = data[xslice, yslice, zslice]

create3D("field3d_1")
create3D("field3d_2")

# Field2D
def create2D_t(name):
Expand All @@ -255,7 +223,6 @@ def create2D_t(name):
result[name] = data[:, xslice, yslice]

create2D_t("field2d_t_1")
create2D_t("field2d_t_2")

def create2D(name):
var = outputfile.createVariable(name, float, ("x", "y"))
Expand All @@ -268,7 +235,6 @@ def create2D(name):
result[name] = data[xslice, yslice]

create2D("field2d_1")
create2D("field2d_2")

# FieldPerp
def createPerp_t(name):
Expand All @@ -283,7 +249,6 @@ def createPerp_t(name):
result[name] = data[:, xslice, zslice]

createPerp_t("fieldperp_t_1")
createPerp_t("fieldperp_t_2")

def createPerp(name):
var = outputfile.createVariable(name, float, ("x", "z"))
Expand All @@ -297,7 +262,6 @@ def createPerp(name):
result[name] = data[xslice, zslice]

createPerp("fieldperp_1")
createPerp("fieldperp_2")

# Time-dependent array
def createScalar_t(name):
Expand All @@ -310,7 +274,6 @@ def createScalar_t(name):

createScalar_t("t_array")
createScalar_t("scalar_t_1")
createScalar_t("scalar_t_2")

# Scalar
def createScalar(name, value):
Expand Down Expand Up @@ -388,7 +351,6 @@ def create3D(name):
result[name] = data[xslice, yslice, zslice]

create3D("field3d_1")
create3D("field3d_2")

# Field2D
def create2D(name):
Expand All @@ -402,7 +364,6 @@ def create2D(name):
result[name] = data[xslice, yslice]

create2D("field2d_1")
create2D("field2d_2")

# FieldPerp
def createPerp(name):
Expand All @@ -417,7 +378,6 @@ def createPerp(name):
result[name] = data[xslice, zslice]

createPerp("fieldperp_1")
createPerp("fieldperp_2")

# Scalar
def createScalar(name, value):
Expand Down Expand Up @@ -466,7 +426,7 @@ def concatenate_data(data_list, *, nxpe, fieldperp_yproc_ind, has_t_dim=True):
raise ValueError("nxpe={} does not divide len(data_list)={}".format(nxpe, npes))

if has_t_dim:
for var in ("field3d_t_1", "field3d_t_2", "field2d_t_1", "field2d_t_2"):
for var in ("field3d_t_1", "field2d_t_1"):
# Join in x-direction
parts = [
np.concatenate(
Expand All @@ -477,7 +437,7 @@ def concatenate_data(data_list, *, nxpe, fieldperp_yproc_ind, has_t_dim=True):
# Join in y-direction
result[var] = np.concatenate(parts, axis=2)

for var in ("field3d_1", "field3d_2", "field2d_1", "field2d_2"):
for var in ("field3d_1", "field2d_1"):
# Join in x-direction
parts = [
np.concatenate(
Expand All @@ -489,7 +449,7 @@ def concatenate_data(data_list, *, nxpe, fieldperp_yproc_ind, has_t_dim=True):
result[var] = np.concatenate(parts, axis=1)

if has_t_dim:
for var in ("fieldperp_t_1", "fieldperp_t_2"):
for var in ("fieldperp_t_1",):
# Join in x-direction
result[var] = np.concatenate(
[
Expand All @@ -501,7 +461,7 @@ def concatenate_data(data_list, *, nxpe, fieldperp_yproc_ind, has_t_dim=True):
axis=1,
)

for var in ("fieldperp_1", "fieldperp_2"):
for var in ("fieldperp_1",):
# Join in x-direction
result[var] = np.concatenate(
[
Expand Down
Loading
Loading