From 3d1c2a1e496c591d3a16571656a72827ee721245 Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Sat, 14 Dec 2024 22:29:30 -0600 Subject: [PATCH 01/12] add spline option for rebinning, make the default --- larch/xafs/rebin_xafs.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/larch/xafs/rebin_xafs.py b/larch/xafs/rebin_xafs.py index cdffe944e..492582b0f 100644 --- a/larch/xafs/rebin_xafs.py +++ b/larch/xafs/rebin_xafs.py @@ -1,4 +1,5 @@ import numpy as np +from scipy.interpolate import CubicSpline from larch import Group, Make_CallArgs, parse_group_args from larch.math import (index_of, interp1d, @@ -54,7 +55,7 @@ def sort_xafs(energy, mu=None, group=None, fix_repeats=True, remove_nans=True, o @Make_CallArgs(["energy", "mu"]) def rebin_xafs(energy, mu=None, group=None, e0=None, pre1=None, pre2=-30, pre_step=2, xanes_step=None, exafs1=15, exafs2=None, - exafs_kstep=0.05, method='centroid'): + exafs_kstep=0.05, method='spline'): """rebin XAFS energy and mu to a 'standard 3 region XAFS scan' Arguments @@ -70,7 +71,7 @@ def rebin_xafs(energy, mu=None, group=None, e0=None, pre1=None, pre2=-30, exafs1 end of XANES region, start of EXAFS region [15] exafs2 end of EXAFS region [last energy point] exafs_kstep k-step for EXAFS region [0.05] - method one of 'boxcar', 'centroid' ['centroid'] + method one of 'spline, 'boxcar', 'centroid' ['spline'] Returns ------- @@ -101,7 +102,8 @@ def rebin_xafs(energy, mu=None, group=None, e0=None, pre1=None, pre2=-30, array. For each new energy bin, the new value is selected from the data in the segment as either a) linear interpolation if there are fewer than 3 points in the segment. - b) mean value ('boxcar') + b) spline interpolation ('spline') + c) mean value ('boxcar') c) centroid ('centroid') """ @@ -173,6 +175,8 @@ def rebin_xafs(energy, mu=None, group=None, e0=None, pre1=None, pre2=-30, else: if method.startswith('box'): val = mu[j0:j1].mean() + elif method.startswith('spl'): + val = CubicSpline(energy[j0:j1], mu[j0:j1])(en[i]) else: val = (mu[j0:j1]*energy[j0:j1]).mean()/energy[j0:j1].mean() mu_out.append(val) From 1123f02a602847bf25642fec2d93972872b0dc77 Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Sat, 14 Dec 2024 22:30:13 -0600 Subject: [PATCH 02/12] add option for smoothing method in rebinning xafs --- larch/wxxas/xas_dialogs.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/larch/wxxas/xas_dialogs.py b/larch/wxxas/xas_dialogs.py index 323c1cbdf..d9ab7f3a5 100644 --- a/larch/wxxas/xas_dialogs.py +++ b/larch/wxxas/xas_dialogs.py @@ -580,8 +580,10 @@ def __init__(self, parent, controller, **kws): wids['xanes_step'] = FloatCtrl(panel, value=xanes_step, **opts) wids['exafs_step'] = FloatCtrl(panel, value=0.05, **opts) + wids['method'] = Choice(panel, choices=('spline', 'boxcar', 'centroid'), + size=(80, -1), action=self.on_rebin) for wname, wid in wids.items(): - if wname != 'grouplist': + if wname not in ('grouplist', 'method'): wid.SetAction(partial(self.on_rebin, name=wname)) #wids['apply'] = Button(panel, 'Save / Overwrite', size=(150, -1), @@ -592,7 +594,7 @@ def __init__(self, parent, controller, **kws): action=self.on_saveas) SetTip(wids['save_as'], 'Save corrected data as new group') - wids['save_as_name'] = wx.TextCtrl(panel, -1, self.dgroup.filename + '_rebin', + wids['save_as_name'] = wx.TextCtrl(panel, -1, self.dgroup.filename + '_rebin_s', size=(250, -1)) def add_text(text, dcol=1, newrow=True): @@ -629,6 +631,9 @@ def add_text(text, dcol=1, newrow=True): panel.Add(wids['exafs_step']) add_text('1/\u212B', newrow=False) + add_text('Smoothing Method: ', dcol=2) + panel.Add(wids['method']) + # panel.Add(wids['apply'], dcol=2, newrow=True) panel.Add(wids['save_as'], dcol=2, newrow=True) panel.Add(wids['save_as_name'], dcol=3) @@ -664,8 +669,10 @@ def on_rebin(self, event=None, name=None, value=None): val = wids['exafs1'].GetValue() wids['xanes2'].SetValue(ktoe(val), act=False) + method = wids['method'].GetStringSelection().lower() e0 = wids['e0'].GetValue() args = dict(group=self.dgroup.groupname, e0=e0, + method=method, pre1=wids['pre1'].GetValue(), pre2=wids['pre2'].GetValue(), pre_step=wids['pre_step'].GetValue(), @@ -677,9 +684,10 @@ def on_rebin(self, event=None, name=None, value=None): # do rebin: cmd = """rebin_xafs({group}, e0={e0:f}, pre1={pre1:f}, pre2={pre2:f}, pre_step={pre_step:f}, xanes_step={xanes_step:f}, exafs1={exafs1:f}, - exafs2={exafs2:f}, exafs_kstep={exafs_kstep:f})""".format(**args) + exafs2={exafs2:f}, exafs_kstep={exafs_kstep:f}, method='{method}')""".format(**args) self.cmd = cmd self.controller.larch.eval(cmd) + wids['save_as_name'].SetValue(f'{self.dgroup.filename}_rebin_{method[:3]}') if hasattr(self.dgroup, 'rebinned'): xnew = self.dgroup.rebinned.energy From 814b027d9baa7e3b16264a072ac595d8790fde6a Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Mon, 16 Dec 2024 21:49:42 -0600 Subject: [PATCH 03/12] add R^2 to fit stats for pre-edge peak fitting --- larch/wxxas/prepeak_panel.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/larch/wxxas/prepeak_panel.py b/larch/wxxas/prepeak_panel.py index 11305925c..eea7742c9 100644 --- a/larch/wxxas/prepeak_panel.py +++ b/larch/wxxas/prepeak_panel.py @@ -302,18 +302,19 @@ def build(self): sview.SetFont(self.font_fixedwidth) - xw = (175, 85, 85, 130, 130, 130) + xw = (170, 75, 75, 110, 115, 115, 100) if uname=='darwin': - xw = (160, 75, 75, 110, 110, 120) + xw = (150, 70, 70, 90, 95, 95, 95) sview.Bind(dv.EVT_DATAVIEW_SELECTION_CHANGED, self.onSelectFit) sview.AppendTextColumn('Label', width=xw[0]) - sview.AppendTextColumn('N_data', width=xw[1]) - sview.AppendTextColumn('N_vary', width=xw[2]) + sview.AppendTextColumn('Ndata', width=xw[1]) + sview.AppendTextColumn('Nvary', width=xw[2]) sview.AppendTextColumn('\u03c7\u00B2', width=xw[3]) sview.AppendTextColumn('reduced \u03c7\u00B2', width=xw[4]) sview.AppendTextColumn('Akaike Info', width=xw[5]) + sview.AppendTextColumn('R^2', width=xw[6]) for col in range(sview.ColumnCount): this = sview.Columns[col] @@ -321,7 +322,7 @@ def build(self): this.Alignment = wx.ALIGN_RIGHT if col > 0 else wx.ALIGN_LEFT this.Renderer.Alignment = this.Alignment - sview.SetMinSize((750, 150)) + sview.SetMinSize((775, 150)) irow += 1 sizer.Add(sview, (irow, 0), (1, 5), LEFT) @@ -357,7 +358,7 @@ def build(self): this.Alignment = wx.ALIGN_RIGHT if col in (1, 2) else wx.ALIGN_LEFT this.Renderer.Alignment = this.Alignment - pview.SetMinSize((750, 200)) + pview.SetMinSize((775, 200)) pview.Bind(dv.EVT_DATAVIEW_SELECTION_CHANGED, self.onSelectParameter) irow += 1 @@ -466,7 +467,7 @@ def onSaveAllStats(self, evt=None): labels = [('Data Set' + ' '*25)[:25], 'Group name', 'n_data', 'n_varys', 'chi-square', 'reduced_chi-square', - 'akaike_info', 'bayesian_info'] + 'akaike_info', 'bayesian_info', 'R^2'] for pname in param_names: labels.append(pname) @@ -485,7 +486,7 @@ def onSaveAllStats(self, evt=None): label = (label + ' '*25)[:25] dat = [label, dgroup.groupname, '%d' % result.ndata, '%d' % result.nvarys] - for attr in ('chisqr', 'redchi', 'aic', 'bic'): + for attr in ('chisqr', 'redchi', 'aic', 'bic', 'rsquared'): dat.append(gformat(getattr(result, attr), 11)) for pname in param_names: val = stderr = 0 @@ -636,16 +637,18 @@ def show_results(self, datagroup=None, form=None, show_plot=False, larch_eval=No datagroup = self.datagroup self.peakfit_history = getattr(self.datagroup.prepeaks, 'fit_history', []) - # cur = self.get_fitresult() wids = self.wids wids['stats'].DeleteAllItems() for i, res in enumerate(self.peakfit_history): args = [res.label] - for attr in ('ndata', 'nvarys', 'chisqr', 'redchi', 'aic'): + for attr in ('ndata', 'nvarys', 'chisqr', 'redchi', + 'aic', 'rsquared'): val = getattr(res.result, attr) if isinstance(val, int): val = '%d' % val + elif attr == 'rsquared': + val = f"{val:.5f}" else: val = gformat(val, 10) args.append(val) @@ -1044,6 +1047,7 @@ def fill_model_params(self, prefix, params): pname = prefix + pname if pname in parwids: wids = parwids[pname] + wids.value.SetValue(par.value) if wids.minval is not None: wids.minval.SetValue(par.min) if wids.maxval is not None: @@ -1053,7 +1057,7 @@ def fill_model_params(self, prefix, params): varstr = 'constrain' if wids.vary is not None: wids.vary.SetStringSelection(varstr) - wids.value.SetValue(par.value) + def onPlotModel(self, evt=None): dgroup = self.controller.get_group() From 677e32c84d97899336d3c4c9f779835d8a076494 Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Mon, 16 Dec 2024 21:50:30 -0600 Subject: [PATCH 04/12] avoid system bell on invalid floats for parameter widgits --- larch/wxlib/parameter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/larch/wxlib/parameter.py b/larch/wxlib/parameter.py index cfad1ae03..398a5f2b5 100644 --- a/larch/wxlib/parameter.py +++ b/larch/wxlib/parameter.py @@ -103,6 +103,7 @@ def __init__(self, parent, param, name_size=None, prefix=None, maxval=param.max, action=self.onValue, act_on_losefocus=True, + bell_on_invalid=False, gformat=True, size=(float_size, -1)) self.widgets.append(self.value) @@ -325,6 +326,7 @@ def __init__(self, parent, param, precision=4, vary=None, self.wids.val = FloatCtrl(panel, value=param.value, size=(100, -1), precision=precision, + bell_on_invalid=False, minval=minval, maxval=maxval) self.wids.min = FloatCtrl(panel, value=minval, size=(100, -1)) self.wids.max = FloatCtrl(panel, value=maxval, size=(100, -1)) @@ -392,6 +394,7 @@ def __init__(self, parent, param, size=(80, -1), show_name=False, self.wids.val = FloatCtrl(self, value=param.value, minval=param.min, maxval=param.max, + bell_on_invalid=False, precision=precision, size=size) self.wids.name = None From b7f176cecb8d905d0568e63f803c3f99a4f57f43 Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Fri, 20 Dec 2024 12:09:08 -0600 Subject: [PATCH 05/12] allow semicolon- or tab-delimited lines as well as space- and comma-delimited --- larch/io/columnfile.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/larch/io/columnfile.py b/larch/io/columnfile.py index 795d00608..a7b75ca92 100644 --- a/larch/io/columnfile.py +++ b/larch/io/columnfile.py @@ -105,7 +105,11 @@ def getfloats(txt, allow_times=True): Unix timestamp, using time.mktime(dateutil.parser.parse(word).timetuple()) """ - words = [w.strip() for w in txt.replace(',', ' ').split()] + t = txt[:] + for delim in ('\t', ',', ';'): + if t.count(delim) > 0: + t = t.replace(delim, ' ') + words = [w.strip() for w in t.split()] mktime = time.mktime for i, w in enumerate(words): val = None @@ -278,7 +282,7 @@ def read_ascii(filename, labels=None, simple_labels=False, Examples: >>> feo_data = read_ascii('feo_rt1.dat') - >>> show(g)a + >>> show(g) == Group ascii_file feo_rt1.dat: 0 methods, 8 attributes == array_labels: ['energy', 'xmu', 'i0'] attrs: From 3ad4e83829fb44672c36500cc85c1f53514efed5 Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Fri, 20 Dec 2024 12:32:37 -0600 Subject: [PATCH 06/12] more improvements to reading data lines, including respecting allow_dates --- larch/io/columnfile.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/larch/io/columnfile.py b/larch/io/columnfile.py index a7b75ca92..55f985ab1 100644 --- a/larch/io/columnfile.py +++ b/larch/io/columnfile.py @@ -103,23 +103,26 @@ def getfloats(txt, allow_times=True): The `allow_times` will try to support common date-time strings using the dateutil module, returning a numerical value as the Unix timestamp, using - time.mktime(dateutil.parser.parse(word).timetuple()) + dateutil.parser.parse(word).timestamp() """ - t = txt[:] + t = txt[:].strip() + if t[0] in ('#', ';', '!', '<', '*', '%'): + return [None] + for delim in ('\t', ',', ';'): if t.count(delim) > 0: t = t.replace(delim, ' ') words = [w.strip() for w in t.split()] - mktime = time.mktime for i, w in enumerate(words): val = None try: val = float(w) except ValueError: - try: - val = mktime(dateparse(w).timetuple()) - except ValueError: - pass + if allow_times: + try: + val = dateparse(w).timestamp() + except ValueError: + pass words[i] = val return words From fb54fe65963685eb7239b86c1dfc11653ad1654e Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Fri, 20 Dec 2024 12:34:07 -0600 Subject: [PATCH 07/12] make sure multi-column imports sets 'mu' array --- larch/wxxas/xasgui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/larch/wxxas/xasgui.py b/larch/wxxas/xasgui.py index cea697a71..efafc3b47 100644 --- a/larch/wxxas/xasgui.py +++ b/larch/wxxas/xasgui.py @@ -1449,7 +1449,7 @@ def onReadSpecfile_OK(self, script, path, scanlist, config=None): mscript = '\n'.join(["{ngroup} = deepcopy({group})", yplotline, "{ngroup}.mu = {ngroup}.yplot", - "{ngroup}.plot_ylabel = '{ylabel}'"]) + "{ngroup}.plot_ylabel = '{ylabel}'"]) i0 = '1.0' if multi_i0 < len(config['array_labels']): i0 = config['array_labels'][multi_i0] @@ -1700,7 +1700,7 @@ def install_multichans(config): yplotline = line.replace("{group}", "{ngroup}") mscript = ["{ngroup} = deepcopy({group})", yplotline, - "{ngroup}.{yarray} = {ngroup}.yplot[:]", + "{ngroup}.mu = {ngroup}.{yarray} = {ngroup}.yplot[:]", "{ngroup}.plot_ylabel = '{ylabel}'" ] if dtype == 'xydata': mscript.append("{ngroup}.scale = ptp({ngroup}.y+1.e-15)") From a1058a856da1d1f72776c54809a9d7604696506a Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Fri, 20 Dec 2024 13:12:23 -0600 Subject: [PATCH 08/12] release 2024.12.0 --- larch/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/larch/version.py b/larch/version.py index d92c8ec43..c942204f9 100644 --- a/larch/version.py +++ b/larch/version.py @@ -1,8 +1,8 @@ #!/usr/bin/env python """Version information""" -__release_version__ = '0.9.81' -__date__ = '2024-September-20' +__release_version__ = '2024.12.0' +__date__ = '2024-December-20' __authors__ = "M. Newville, M. Rovezzi, M. Koker, B. Ravel, and others" from ._version import __version__, __version_tuple__ From fe47313b6fdf1e09bb248f9e5c120e710b33b887 Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Wed, 1 Jan 2025 10:59:04 -0600 Subject: [PATCH 09/12] add remove_nans2() to finde0, which also coerces arrays to numpy arrays --- larch/xafs/pre_edge.py | 1 + 1 file changed, 1 insertion(+) diff --git a/larch/xafs/pre_edge.py b/larch/xafs/pre_edge.py index 9777a1df0..a0acc5609 100644 --- a/larch/xafs/pre_edge.py +++ b/larch/xafs/pre_edge.py @@ -37,6 +37,7 @@ def find_e0(energy, mu=None, group=None, _larch=None): energy, mu, group = parse_group_args(energy, members=('energy', 'mu'), defaults=(mu,), group=group, fcn_name='find_e0') + energy, mu = remove_nans2(energy, mu) # first find e0 without smoothing, then refine with smoothing e1, ie0, estep1 = _finde0(energy, mu, estep=None, use_smooth=False) istart = max(3, ie0-75) From 0535b2731191f250af0226d866734044e3c6ca82 Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Fri, 10 Jan 2025 09:06:13 -0600 Subject: [PATCH 10/12] update installer scripts for latest miniforge --- installers/GetLarch.bat | 2 +- installers/GetLarch.sh | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/installers/GetLarch.bat b/installers/GetLarch.bat index fb82a29e9..4fb249d72 100644 --- a/installers/GetLarch.bat +++ b/installers/GetLarch.bat @@ -19,7 +19,7 @@ echo ## basic mamba installed, running updates set PATH=%prefix%;%prefix%\bin;%prefix%\condabin;%prefix%\Scripts;%PATH% echo ## Installing basic python scipy packages -call %prefix%\Scripts\mamba install -yc conda-forge python==3.11.5 numpy scipy matplotlib h5py scikit-image scikit-learn pandas jupyter plotly wxpython fabio pyfai pymatgen mkl_fft tomopy +call %prefix%\Scripts\mamba install -yc conda-forge python==3.12.7 numpy scipy matplotlib h5py scikit-image scikit-learn pandas jupyter plotly wxpython fabio pyfai pymatgen mkl_fft tomopy echo ## Installing xraylarch and dependencies from PyPI call %prefix%\Scripts\pip install xraylarch[larix] diff --git a/installers/GetLarch.sh b/installers/GetLarch.sh index 7d8ef2efe..5f40efbfb 100644 --- a/installers/GetLarch.sh +++ b/installers/GetLarch.sh @@ -8,19 +8,15 @@ prefix=$HOME/xraylarch larchurl='xraylarch[larix]' uname=`uname` -if [ $uname == Darwin ]; then - uname=MacOSX -fi - condaurl="https://github.com/conda-forge/miniforge/releases/latest/download" condafile="Miniforge3-$uname-x86_64.sh" -condafile="Mambaforge-$uname-x86_64.sh" logfile=GetLarch.log ## set list of conda packages to install from conda-forge -cforge_pkgs="python=>3.12.3 numpy scipy matplotlib h5py scikit-image scikit-learn pandas jupyter plotly wxpython fabio pyfai pymatgen mkl_fft tomopy" +cforge_pkgs="python=>3.12.7 numpy scipy matplotlib h5py scikit-image scikit-learn pandas jupyter plotly wxpython fabio pyfai pymatgen mkl_fft tomopy" + unset CONDA_EXE CONDA_PYTHON_EXE CONDA_PREFIX PROJ_LIB From 79d39abb4bdd4350157b751766bdf0eb31ee1d15 Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Fri, 10 Jan 2025 09:14:09 -0600 Subject: [PATCH 11/12] force control and print configuration for running feff --- larch/wxlib/cif_browser.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/larch/wxlib/cif_browser.py b/larch/wxlib/cif_browser.py index 1444599a8..db4a03905 100644 --- a/larch/wxlib/cif_browser.py +++ b/larch/wxlib/cif_browser.py @@ -557,6 +557,17 @@ def onGetFeff(self, event=None): fefftext = cif2feffinp(cif.ciftext, catom, edge=edge, cluster_size=csize, absorber_site=site_index, version8=version8, with_h=with_h, extra_titles=etitles) + # hack for larixite 2024.10.0, + # but also, here we really want to force the print and control flags + flines = fefftext.split('\n') + for i, l in enumerate(flines): + x = l.strip() + if x.startswith('PRINT'): + flines[i] = 'PRINT 1 0 0 0 0 3' + elif x.startswith('CONTROL'): + flines[i] = 'CONTROL 1 1 1 1 1 1' + + fefftext = '\n'.join(flines) self.wids['feff_runfolder'].SetValue(folder) self.wids['feff_text'].SetValue(fefftext) From 58e209d79d9d689a495d455e8bee1b3e1fcad033 Mon Sep 17 00:00:00 2001 From: Matthew Newville Date: Fri, 10 Jan 2025 10:05:11 -0600 Subject: [PATCH 12/12] update dependencies, constructor script --- installers/conda_constructor/construct.yaml | 8 ++++---- setup.cfg | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/installers/conda_constructor/construct.yaml b/installers/conda_constructor/construct.yaml index f687b0542..c1071c91a 100644 --- a/installers/conda_constructor/construct.yaml +++ b/installers/conda_constructor/construct.yaml @@ -1,5 +1,5 @@ name: xraylarch -version: 2023-10 +version: 2025-01 channels: - https://conda.anaconda.org/conda-forge/ @@ -18,15 +18,15 @@ post_install: post_install_posix.sh [linux or osx] post_install: post_install_windows.bat [win] specs: - - python==3.11.5 + - python==3.12.8 - conda - mamba - openssl - setuptools>=61.0 - pip - requests - - numpy>=1.24 - - scipy>=1.9 + - numpy>=1.25 + - scipy>=1.13 - matplotlib>=3.6 - h5py>=3.7 - sqlalchemy>=2.0 diff --git a/setup.cfg b/setup.cfg index 3dbde4ea4..430575093 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,13 +35,13 @@ include_package_data = True python_requires = >=3.9 setup_requires = setuptools_scm install_requires = - numpy>=1.22,<2 - scipy>=1.7 + numpy>=1.25 + scipy>=1.13 lmfit>=1.3.1 - asteval>=1.0.2 + asteval>=1.0.4 uncertainties>=3.2.1 pyshortcuts>=1.9.5 - xraydb>=4.5.3 + xraydb>=4.5.5 larixite silx>=0.15.2 matplotlib>=3.5