From e013a836596f707421d1580ba8d76a2f0066571a Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Tue, 4 Aug 2026 15:17:09 -0400 Subject: [PATCH 01/13] Update examples to use the correct parser and add description of how to create your own parser in docs --- docs/examples/coreshellnp.py | 7 ++- docs/examples/crystalpdf.py | 11 ++--- docs/examples/crystalpdfall.py | 7 ++- docs/examples/crystalpdfobjcryst.py | 9 ++-- docs/examples/crystalpdftwodata.py | 11 ++--- docs/examples/crystalpdftwophase.py | 7 ++- docs/examples/ellipsoidsas.py | 2 +- docs/examples/nppdfcrystal.py | 7 ++- docs/examples/nppdfsas.py | 9 ++-- docs/source/extending.rst | 73 +++++++++++++++++++++-------- 10 files changed, 84 insertions(+), 59 deletions(-) diff --git a/docs/examples/coreshellnp.py b/docs/examples/coreshellnp.py index f2f2637c..fd54a50f 100644 --- a/docs/examples/coreshellnp.py +++ b/docs/examples/coreshellnp.py @@ -29,9 +29,8 @@ FitRecipe, FitResults, Profile, - ProfileParser, ) -from diffpy.srfit.pdf import PDFGenerator +from diffpy.srfit.pdf import PDFGenerator, PDFParser # Example Code @@ -42,8 +41,8 @@ def makeRecipe(stru1, stru2, datname): profile = Profile() # Load data and add it to the profile - parser = ProfileParser() - parser.parseFile(datname) + parser = PDFParser() + parser.parse_file(datname) profile.load_parsed_data(parser) profile.set_calculation_range(xmin=1.5, xmax=45, dx=0.1) diff --git a/docs/examples/crystalpdf.py b/docs/examples/crystalpdf.py index b378ff7d..bb6ad3cf 100644 --- a/docs/examples/crystalpdf.py +++ b/docs/examples/crystalpdf.py @@ -32,9 +32,8 @@ FitRecipe, FitResults, Profile, - ProfileParser, ) -from diffpy.srfit.pdf import PDFGenerator +from diffpy.srfit.pdf import PDFGenerator, PDFParser from diffpy.structure import Structure ###### @@ -48,13 +47,13 @@ def makeRecipe(ciffile, datname): profile = Profile() # Load data and add it to the Profile. Unlike in other examples, we use a - # class (ProfileParser) to help us load the data. This class will read the + # class (PDFParser) to help us load the data. This class will read the # data and relevant metadata from a two- to four-column data file generated # with PDFGetX2 or PDFGetN. The metadata will be passed to the PDFGenerator # when they are associated in the FitContribution, which saves some # configuration steps. - parser = ProfileParser() - parser.parseFile(datname) + parser = PDFParser() + parser.parse_file(datname) profile.load_parsed_data(parser) profile.set_calculation_range(xmax=20) @@ -63,7 +62,7 @@ def makeRecipe(ciffile, datname): # we want to refine a Structure object from diffpy.structure. We tell the # PDFGenerator that with the 'setStructure' method. All other configuration # options will be inferred from the metadata that is read by the - # ProfileParser. + # PDFParser. # In particular, this will set the scattering type (x-ray or neutron), the # Qmax value, as well as initial values for the non-structural Parameters. generator = PDFGenerator("G") diff --git a/docs/examples/crystalpdfall.py b/docs/examples/crystalpdfall.py index 71d8da0f..dc2da591 100644 --- a/docs/examples/crystalpdfall.py +++ b/docs/examples/crystalpdfall.py @@ -27,9 +27,8 @@ FitRecipe, FitResults, Profile, - ProfileParser, ) -from diffpy.srfit.pdf import PDFGenerator +from diffpy.srfit.pdf import PDFGenerator, PDFParser ###### # Example Code @@ -38,8 +37,8 @@ def makeProfile(datafile): """Make an place data within a Profile.""" profile = Profile() - parser = ProfileParser() - parser.parseFile(datafile) + parser = PDFParser() + parser.parse_file(datafile) profile.load_parsed_data(parser) profile.set_calculation_range(xmax=20) return profile diff --git a/docs/examples/crystalpdfobjcryst.py b/docs/examples/crystalpdfobjcryst.py index 3f4787f5..0d427d04 100644 --- a/docs/examples/crystalpdfobjcryst.py +++ b/docs/examples/crystalpdfobjcryst.py @@ -28,9 +28,8 @@ FitRecipe, FitResults, Profile, - ProfileParser, ) -from diffpy.srfit.pdf import PDFGenerator +from diffpy.srfit.pdf import PDFGenerator, PDFParser ###### # Example Code @@ -42,12 +41,12 @@ def makeRecipe(ciffile, datname): # This will be used to store the observed and calculated PDF profile. profile = Profile() - # Load data and add it to the Profile. As before we use a ProfileParser. + # Load data and add it to the Profile. As before we use a PDFParser. # The metadata is still passed to the PDFGenerator later on. # The interaction between the PDFGenerator and the metadata does not # depend on type of structure being refined. - parser = ProfileParser() - parser.parseFile(datname) + parser = PDFParser() + parser.parse_file(datname) profile.load_parsed_data(parser) profile.set_calculation_range(xmax=20) diff --git a/docs/examples/crystalpdftwodata.py b/docs/examples/crystalpdftwodata.py index 57dc9d12..8c9eafe6 100644 --- a/docs/examples/crystalpdftwodata.py +++ b/docs/examples/crystalpdftwodata.py @@ -29,9 +29,8 @@ FitRecipe, FitResults, Profile, - ProfileParser, ) -from diffpy.srfit.pdf import PDFGenerator +from diffpy.srfit.pdf import PDFGenerator, PDFParser ###### # Example Code @@ -46,13 +45,13 @@ def makeRecipe(ciffile, xdatname, ndatname): nprofile = Profile() # Load data and add it to the proper Profile. - parser = ProfileParser() - parser.parseFile(xdatname) + parser = PDFParser() + parser.parse_file(xdatname) xprofile.load_parsed_data(parser) xprofile.set_calculation_range(xmax=20) - parser = ProfileParser() - parser.parseFile(ndatname) + parser = PDFParser() + parser.parse_file(ndatname) nprofile.load_parsed_data(parser) nprofile.set_calculation_range(xmax=20) diff --git a/docs/examples/crystalpdftwophase.py b/docs/examples/crystalpdftwophase.py index b62945c7..9407d028 100644 --- a/docs/examples/crystalpdftwophase.py +++ b/docs/examples/crystalpdftwophase.py @@ -29,9 +29,8 @@ FitRecipe, FitResults, Profile, - ProfileParser, ) -from diffpy.srfit.pdf import PDFGenerator +from diffpy.srfit.pdf import PDFGenerator, PDFParser ###### # Example Code @@ -43,8 +42,8 @@ def makeRecipe(niciffile, siciffile, datname): profile = Profile() # Load data and add it to the profile - parser = ProfileParser() - parser.parseFile(datname) + parser = PDFParser() + parser.parse_file(datname) profile.load_parsed_data(parser) profile.set_calculation_range(xmax=20) diff --git a/docs/examples/ellipsoidsas.py b/docs/examples/ellipsoidsas.py index 58bb0bb3..b1fa1aa3 100644 --- a/docs/examples/ellipsoidsas.py +++ b/docs/examples/ellipsoidsas.py @@ -37,7 +37,7 @@ def makeRecipe(datname): # Load data and add it to the Profile. We use a SASParser to load the data # properly and pass the metadata along. parser = SASParser() - parser.parseFile(datname) + parser.parse_file(datname) profile.load_parsed_data(parser) # The ProfileGenerator diff --git a/docs/examples/nppdfcrystal.py b/docs/examples/nppdfcrystal.py index 39f12354..033a7162 100644 --- a/docs/examples/nppdfcrystal.py +++ b/docs/examples/nppdfcrystal.py @@ -32,9 +32,8 @@ FitRecipe, FitResults, Profile, - ProfileParser, ) -from diffpy.srfit.pdf import PDFGenerator +from diffpy.srfit.pdf import PDFGenerator, PDFParser def makeRecipe(ciffile, grdata): @@ -42,8 +41,8 @@ def makeRecipe(ciffile, grdata): # Set up a PDF fit as has been done in other examples. pdfprofile = Profile() - pdfparser = ProfileParser() - pdfparser.parseFile(grdata) + pdfparser = PDFParser() + pdfparser.parse_file(grdata) pdfprofile.load_parsed_data(pdfparser) pdfprofile.set_calculation_range(xmin=0.1, xmax=20) diff --git a/docs/examples/nppdfsas.py b/docs/examples/nppdfsas.py index a4a54a6f..863fea7a 100644 --- a/docs/examples/nppdfsas.py +++ b/docs/examples/nppdfsas.py @@ -31,9 +31,8 @@ FitRecipe, FitResults, Profile, - ProfileParser, ) -from diffpy.srfit.pdf import PDFGenerator +from diffpy.srfit.pdf import PDFGenerator, PDFParser from diffpy.srfit.pdf.characteristicfunctions import SASCF from diffpy.srfit.sas import SASGenerator, SASParser @@ -47,8 +46,8 @@ def makeRecipe(ciffile, grdata, iqdata): """ # Create a PDF contribution as before pdfprofile = Profile() - pdfparser = ProfileParser() - pdfparser.parseFile(grdata) + pdfparser = PDFParser() + pdfparser.parse_file(grdata) pdfprofile.load_parsed_data(pdfparser) pdfprofile.set_calculation_range(xmin=0.1, xmax=20) @@ -66,7 +65,7 @@ def makeRecipe(ciffile, grdata, iqdata): # elliptical. sasprofile = Profile() sasparser = SASParser() - sasparser.parseFile(iqdata) + sasparser.parse_file(iqdata) sasprofile.load_parsed_data(sasparser) if all(sasprofile.dy == 0): sasprofile.dy[:] = 1 diff --git a/docs/source/extending.rst b/docs/source/extending.rst index 164ef187..2c6862e6 100644 --- a/docs/source/extending.rst +++ b/docs/source/extending.rst @@ -122,29 +122,63 @@ functions as in the second example. Extending Profile Parsers -------------------------- -The ``ProfileParser`` class is located in the ``diffpy.srfit.fitbase.parser`` -module. The purpose of this class is to read data and metadata from a file or -string and pass those data and metadata to a ``Profile`` instance. The -``Profile`` in turn will pass this information to a ``ProfileGenerator``. +The ``ProfileParser`` class is located in the +``diffpy.srfit.fitbase.profileparser`` module. The purpose of this class is to +read data and metadata from a file and pass those data and metadata to a +``Profile`` instance. The ``Profile`` in turn will pass this information to a +``ProfileGenerator``. -The simplest way to extend the ``ProfileParser`` is to derive a new class from -``ProfileParser`` and overload the ``parseString`` method. By default, the -``parseFile`` method can read an ASCII file and passes the loaded string to the -``parseString`` method. For non-ASCII data one should overload both of these -methods. An example of a customized ``ProfileParser`` is the ``PDFParser`` -class in the ``diffpy.srfit.pdf.pdfparser`` module. +``parse_file`` is a template method that a subclass is not expected to +override. It resets the parser, calls the two hooks described below, records +the file name and selects the first bank. A format is customized by overriding +one or both hooks: -Here is a simple example demonstrating how to extract (x,y) data from a -two-column string. :: +``_parse_metadata(filename)`` + Return the metadata read from the header, as a dictionary. The default + implementation uses ``load_data`` from ``diffpy.utils.parsers`` to collect + plain ``name = value`` pairs. - def parseString(self, datastring): +``_parse_data(filename, column_format=None, **kwargs)`` + Append one entry to ``self._banks`` for each data set in the file. The + default implementation reads a single bank with ``load_data`` and maps its + columns onto ``x``, ``y``, ``dx`` and ``dy``. + +An example of a customized ``ProfileParser`` is the ``PDFParser`` class in the +``diffpy.srfit.pdf.pdfparser`` module. It overrides ``_parse_metadata`` alone, +so that PDFgetX and PDFgetN headers yield ``stype``, ``qmin``, ``qmax`` and the +other PDF specific values, while the column handling is inherited unchanged. + +Here is a simple example demonstrating how to read metadata that is stored as +``name: value`` pairs rather than the default ``name = value``. :: + + def _parse_metadata(self, filename): + + meta = {} + + for line in Path(filename).read_text().splitlines(): + if not line.startswith("#"): + break + name, sep, value = line.lstrip("# ").partition(":") + if sep: + meta[name.strip()] = value.strip() + + return meta + +The parser can put any information into the returned dictionary; it is merged +into the ``_meta`` attribute. It is up to a ``ProfileGenerator`` that may use +the parsed data to define and retrieve usable metadata. + +A format whose data block is not a plain matrix of columns overrides +``_parse_data`` instead. :: + + def _parse_data(self, filename, column_format=None, **kwargs): xvals = [] yvals = [] dxvals = None dyvals = None - for line in datastring.splitlines(): + for line in Path(filename).read_text().splitlines(): sline = line.split() x, y = map(float, sline) @@ -158,16 +192,15 @@ The ``self._banks.append`` line puts the data arrays into the ``_banks`` list. This list is for collecting multiple data sets that may be present within a single file. The ``dxvals`` and ``dyvals`` are the uncertainty values on the ``xvals`` and ``yvals``. In this simple example they are not present, and so -are set to None. - -In general, the data string may contain metadata. The ``ProfileParser`` has a -dictionary attribute named ``_meta``. The parser can put any information into -this dictionary. It is up to a ``ProfileGenerator`` that may use the parsed -data to define and retrieve usable metadata. +are set to None. A ``Profile`` reads that as an unweighted data set. If the data are not in a form that can be stored in a ``Profile`` then it is the responsibility of the parser to convert this data to a usable form. +Parsers read from a file, not from a string. The older ``parseString`` method +has been removed, and ``parseFile`` is deprecated: it now delegates to +``parse_file`` and will be removed in version 4.0.0. + Extending Profiles -------------------------- From b39c20b6e1e5072f5f30ebb9c51e755f52baca55 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Tue, 4 Aug 2026 15:17:56 -0400 Subject: [PATCH 02/13] update old metadata format to new xPDFsuite and pdfgetx headers. One testdata file for each --- tests/testdata/ni-q27r100-neutron.gr | 70 +++++-------- tests/testdata/si-q27r60-xray.gr | 142 ++++----------------------- 2 files changed, 41 insertions(+), 171 deletions(-) diff --git a/tests/testdata/ni-q27r100-neutron.gr b/tests/testdata/ni-q27r100-neutron.gr index 1b18d9a1..c602b50b 100644 --- a/tests/testdata/ni-q27r100-neutron.gr +++ b/tests/testdata/ni-q27r100-neutron.gr @@ -1,50 +1,26 @@ -# History written: Tue May 6 11:04:33 2008 -# produced by bozin -# ##### Run Information runCorrection=T -# prep=gsas machine=npdf -# run=npdf_03315 background=npdf_03001 -# smooth=2 smoothParam=32 32 0 backKillThresh=-1.0 -# in beam: radius=0.45325 height=4.5 -# temp=300 runTitle=Run 3315: Ni commercial, RT_stick -# -# ##### Vanadium runCorrection=T -# run=npdf_03000 background=npdf_03001 -# smooth=2 smoothParam=32 32 0 vanKillThresh=-1.0 vBackKillThresh=-1.0 -# in beam: radius=0.47625 height=4.5 -# -# ##### Container runCorrection=T -# run=npdf_03002 background=npdf_03001 -# smooth=2 smoothParam=32 32 0 cBackKillThresh=-1.0 -# wallThick=0.023 atomDensity=0.072110 -# atomic information: scattCS=5.100 absorpCS=5.080 -# -# ##### Sample Material numElements=1 NormLaue=0.00000 -# Element relAtomNum atomMass atomCoherCS atomIncoherCS atomAbsorpCS -# Ni 1.0000 58.693 13.3000 5.2000 4.49000 -# density=7.0 effDensity=2.8777 -# -# ##### Banks=4 deltaQ=0.01 matchRef=0 matchScal=T matchOffset=T -# bank angle blendQmin blendQmax (0.0 means no info) -# 1 46.6 0.87 21.63 -# 2 90.0 1.51 37.66 -# 3 119.0 1.84 45.96 -# 4 148.0 2.05 51.25 -# -# ##### Program Specific Information -# ## Ft calcError=1 (1 for true, 0 for false) -# numRpoints=10000 maxR=100.0 numDensity=0.0 intMaxR=1.5 -# ## Damp Qmin=0.87 Qmax=27.0 startDampQ=27.0 QAveMin=0.6 -# dampFuncType=0 modEqn=1.0000*S(Q) +0.0000 +0.0000*Q dampExtraToZero=0 -# ## Blend numBanks=4 banks=1,2,3,4 -# soqCorrFile= -# ## Soqd minProcOut=0 -# samPlazcek=1 vanPlazcek=1 smoothData=0 modifyData=1 -# ## Corps minProcOut=0 numBanksMiss=0 -# -# ##### prepgsas prepOutput=1 numBanksMiss=0 fileExt=gsa -# instParamFile=npdf_TL-displex_2018.iparm -# numBanksAdd=0 -# numBanksMult=0 +# xPDFsuite Configuration # +[PDF] +wavelength = 1.333 +dataformat = QA +inputfile = npdf_03315.chi +backgroundfile = npdf_03001.chi +mode = neutron +bgscale = 1.0 +composition = Ni +outputtype = gr +qmaxinst = 27.0 +qmin = 0.87 +qmax = 27.0 +temperature = 300 +rmax = 100.0 +rmin = 0.0 +rstep = 0.01 +rpoly = 0.9 + +[Misc] +inputdir = /data/npdf/chi +savedir = /data/npdf/gr + ##### start data #O0 rg_int sig_rg_int low_int sig_low_int rmax rhofit #S 1 - PDF from PDFgetN diff --git a/tests/testdata/si-q27r60-xray.gr b/tests/testdata/si-q27r60-xray.gr index a0f208ad..d51df93d 100644 --- a/tests/testdata/si-q27r60-xray.gr +++ b/tests/testdata/si-q27r60-xray.gr @@ -1,132 +1,26 @@ -History written: Mon Apr 21 20:48:28 2008 -Produced by -####### Get_XPDF ####### +[DEFAULT] -##### General_Setting -title=X-ray PDF -workingdirectory=e:\Ahmad\MUCAT0804\standards\pdfgetx2 -sourcedir=C:\Program Files\PDFgetX2\ -logfile=.pdfgetx2.log -quiet=0 debug=0 autosave_isa=1 savefilenamebase=si325_mesh_300k_nor_4-8 -iqfilesurfix=.iq sqfilesurfix=.sq fqfilesurfix=.fq grfilesurfix=.gr -stype = X -qmax = 27 -temperature = 300 -##### DataFileFormat -datatype=1 (0:SPEC, 1:CHI, 2:nxm column, 3:unknown) -num_skiplines=3 comment_id=# delimiter= -### SPEC Format scan_id=#S scan_delimiter= -columnname_id=#L columnname_delimiter= -data_id= data_delimiter= -### CHI Format -### nxm column Format -### End of file format - -##### Data&Background -samfile=si325_mesh_300k_nor_4-8.chi num_sams=1 -sambkgfile=kapton_bgrd_300k_nor_2-3.chi num_sambkgs=1 -confile= num_cons=1 -conbkgfile= num_conbkgs=1 -det# used xcol detcol deterrcol xmin xmax add_det mul_det add_bkg mul_bkg add_con mul_con add_conbkg mul_conbkg - 0 1 0 1 3 0.600000 32.0000 0.000000 1.00000 0.000000 1.00000 0.000000 1.00000 0.000000 1.00000 - -##### Experiment_Setup -title=PDF analysis -user=me -facility=In house -temperature=300.000 containermut=0.000500000 filtermut=0.0200000 -## X-Ray radiationtype=3 - (0: Ag K_alpha, 1:Cu K_alpha, 2:Mo K_alpha, 3:Customize) -lambda=0.142773 energy=86.8406 polartype=0 polardegree=1.00000 -## MonoChromator crystaltype=0 (0:Perfect, 1:Mosaic, 2:None) -position=0 (0:Primary beam, 1:Diffracted beam) -dspacetype=0 (0:Si{111}, 1:Ge{111}, 2:Customize) dspacing=3.13200 - -##### Sample_Setup information num_atoms=1 -#L symbol valence fractions z user_f1 user_f2 user_macoef - Si 0.00 1.000000 14 0.000000 0.000000 0.001000 -geometry=2 mut=0.50000000 numberdensity=0.00600000 -thickness=2.00000 packingFraction=0.500000 theory_mut=0.00579218 +version = diffpy.pdfgetx-2.4.0 -##### GetIQ_Setup -xformat=1 -smoothcorr_isa=0 selfnormalize_isa=0 -#L par_name sample sample_bkg container container_bkg -smooth_degree 2 2 2 2 -smooth_width 6 6 6 6 -selfnormalize 0 0 0 0 -filtercorr_isa=0 samfiltercorr_isa=0 sambkgfiltercorr_isa=0 -confiltercorr_isa=0 conbkgfiltercorr_isa=0 -scatveffcorr_isa=1 samconveffcorr_isa=1 sambkgveffcorr_isa=0 -conbkgveffcorr_isa=0 -nonegative_isa=1 negativevalue=-1.00000 +# input and output specifications +dataformat = QA +outputtype = gr -##### Calibration_Data -## Detection efficiency energy dependence detedepxaxis=0 -detedepfunctype=0 detedep_elastic=1.00000 detedep_fluores=1.80000 -detedep_quadra=0.000000 detedep_spline=0.000000 detedep_file= -## Detector transmission energy dependence dettcoefxaxis=0 -dettcoeffunctype=0 dettcoef_elastic=0.950000 dettcoef_fluores=0.600000 -dettcoef_quadra=0.000000 dettcoef_spline=0.000000 dettcoef_file= - -##### IQ_Simulation -### Elastic used_isa=1 mymethod=1 -do_samabsorp=1 do_multscat=1 do_conabsorp=0 do_airabsorp=0 -do_polarization=1 do_oblincident=0 do_energydep=0 -do_breitdirac=0 breitdiracexpo=2.00000 -do_rulandwin=0 rulandwinwidth=0.00100000 -do_useredit=0 add_user=0.000000 mul_user=1.00000 -### Compton used_isa=1 mymethod=1 -do_samabsorp=1 do_multscat=1 do_conabsorp=0 do_airabsorp=0 -do_polarization=1 do_oblincident=0 do_energydep=0 -do_breitdirac=0 breitdiracexpo=2.00000 -do_rulandwin=0 rulandwinwidth=0.00100000 -do_useredit=0 add_user=0.000000 mul_user=1.00000 -### Fluores used_isa=1 mymethod=1 -do_samabsorp=1 do_multscat=1 do_conabsorp=0 do_airabsorp=0 -do_polarization=1 do_oblincident=0 do_energydep=0 -do_breitdirac=0 breitdiracexpo=2.00000 -do_rulandwin=0 rulandwinwidth=0.00100000 -do_useredit=0 add_user=0.000000 mul_user=1.00000 - -##### Correction_Setup corrmethod=0 -oblincident_isa=1 dettranscoef=0.980000 samfluore_isa=1 -samfluoretype=0 samfluorescale=15.000000 -multiscat_isa=1 xraypolar_isa=1 samabsorp_isa=1 -highqscale_isa=1 highqratio=0.600000 scaleconst=0.039181914 -scaleconst_theory=0.039181914 -comptonscat_isa=1 rulandwin_isa=0 rulandintewidth=0.0100000 -comptonmethod=0 breitdirac_isa=1 breitdiracexponent=3 -detefficiency_isa=1 detefficiencytype=2 (0-1: linear, 2-3: quadratic) -detefficiency_a=-0.054826792 detefficiency_b=0.028062565 -lauediffuse_isa=1 -weight_isa=1 weighttype=0 (0: ^2, 1: , 2: Data Smoothed) -weightsmoothrmin=3.00000 weightsmoothwidth=100 weightsmoothcycles=600 -editsq_isa=0 editsqtype=0 add_sq=0.000000 mul_sq=1.00000 -editsqsmoothrmin=3.00000 editsqsmoothwidth=100 editsqsmoothcycles=600 -smoothdata_isa=0 smoothfunctype=0 smoothqmin=12.0000 smoothboxwidth=9 -interpolateqmin_isa=0 qmininterpolationtype=0 -dampfq_isa=0 dampfqtype=0 dampfqwidth=23.0000 +# PDF calculation setup +mode = xray +composition = Si +bgscale = 1.0 +rpoly = 0.9 +qmaxinst = 29 +qmin = 0.01 +qmax = 27 +rmin = 0 +rmax = 60 +rstep = 0.01 +temperature = 300 -##### SqGr_Optimization Setup -ftmethod=0 -## S(q) qmin=0.010000 qmax=27.000000 qgrid=0.000000 -## G(r) rmin=0.010000 rmax=60.000000 rgrid=0.010000 -## SqOptimization sqoptfunction=1 -optqmin=15.0000 optqmax=40.0000 optqgrid=0.000000 -optrmin=0.000000 optrmax=2.20000 optrgrid=0.0200000 -maxiter=20 relstep=0.000000 weighttype=0 weightfunc=0 -fitbkgmult_isa=0 fitsampmut_isa=1 fitpolariz_isa=1 -fitoblique_isa=0 fitfluores_isa=0 -fitrulandw_isa=0 fitenergya_isa=1 fitenergyb_isa=1 -fitsimurulandw_isa=1 fitDetEdepfluores_isa=0 fitDetEdepquadra_isa=0 -fitDetEdepspline_isa=0 fitDetTCoefElastic_isa=0 fitDetTCoefFluores_isa=0 -fitDetTcoefquadra_isa=0 fitDetTcoefspline_isa=0 +# End of config -------------------------------------------------------------- -##### Save&Plot Settings -datatype=GrData iqcorrtype=Int iqsimutype=SimuIq -sqcorrtype=Oblin sqtofqtype=DampFq -gropttype=OptFq miscdatatype=AtomASF ##### start data #F si325_mesh_300k_nor_4-8.gr #D Mon Apr 21 21:17:23 2008 From 251d309d9878539f5a258ee2a7ba58ef0af93989 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Tue, 4 Aug 2026 15:30:41 -0400 Subject: [PATCH 03/13] add test for new parser behavior --- tests/test_fitrecipe.py | 4 +- tests/test_pdf.py | 280 ++++++++++++++++++++++-------------- tests/test_profileparser.py | 127 ++++++++++++++++ tests/test_sas.py | 2 +- 4 files changed, 301 insertions(+), 112 deletions(-) diff --git a/tests/test_fitrecipe.py b/tests/test_fitrecipe.py index 8b62f786..d24ccd2b 100644 --- a/tests/test_fitrecipe.py +++ b/tests/test_fitrecipe.py @@ -23,7 +23,7 @@ from numpy import array_equal, dot, linspace, ones_like, pi, sin from scipy.optimize import leastsq -from diffpy.srfit.fitbase import FitResults, ProfileParser +from diffpy.srfit.fitbase import FitResults from diffpy.srfit.fitbase.fitcontribution import FitContribution from diffpy.srfit.fitbase.fitrecipe import FitRecipe from diffpy.srfit.fitbase.parameter import Parameter @@ -654,7 +654,7 @@ def build_recipe_from_datafile(datafile): """Helper to build a FitRecipe from a datafile using PDFParser and PDFGenerator.""" profile = Profile() - parser = ProfileParser() + parser = PDFParser() parser.parse_file(str(datafile)) profile.load_parsed_data(parser) diff --git a/tests/test_pdf.py b/tests/test_pdf.py index aaeb0cd4..10d818ee 100644 --- a/tests/test_pdf.py +++ b/tests/test_pdf.py @@ -23,7 +23,6 @@ import pytest from diffpy.srfit.exceptions import SrFitError -from diffpy.srfit.fitbase import ProfileParser from diffpy.srfit.fitbase.parameter import Parameter from diffpy.srfit.fitbase.recipeorganizer import RecipeContainer from diffpy.srfit.pdf import PDFContribution, PDFGenerator, PDFParser @@ -31,117 +30,180 @@ # ---------------------------------------------------------------------------- -def testParser1(datafile): - data = datafile("ni-q27r100-neutron.gr") - parser = PDFParser() - parser.parseFile(data) - - meta = parser._meta - - assert data == meta["filename"] - assert 1 == meta["nbanks"] - assert "N" == meta["stype"] - assert 27 == meta["qmax"] - assert 300 == meta.get("temperature") - assert meta.get("qdamp") is None - assert meta.get("qbroad") is None - assert meta.get("spdiameter") is None - assert meta.get("scale") is None - assert meta.get("doping") is None - - x, y, dx, dy = parser.get_data() - assert dx is None - assert dy is None - - testx = numpy.linspace(0.01, 100, 10000) - diff = testx - x - res = numpy.dot(diff, diff) - assert 0 == pytest.approx(res) - - testy = numpy.array( - [ - 1.144, - 2.258, - 3.312, - 4.279, - 5.135, - 5.862, - 6.445, - 6.875, - 7.150, - 7.272, - ] - ) - diff = testy - y[:10] - res = numpy.dot(diff, diff) - assert 0 == pytest.approx(res) - - return +def approx_or_none(expected_values): + """Wrap expected_values in pytest.approx, unless it is None.""" + if expected_values is None: + return None + return pytest.approx(expected_values) -def testParser2(datafile): - data = datafile("si-q27r60-xray.gr") - parser = ProfileParser() - parser.parse_file(data) - - meta = parser._meta - - assert str(data) == meta["filename"] - assert 1 == meta["nbanks"] - assert "X" == meta["stype"] - assert 27 == meta["qmax"] - assert 300 == meta.get("temperature") - assert meta.get("qdamp") is None - assert meta.get("qbroad") is None - assert meta.get("spdiameter") is None - assert meta.get("scale") is None - assert meta.get("doping") is None - - x, y, dx, dy = parser.get_data() - testx = numpy.linspace(0.01, 60, 5999, endpoint=False) - diff = testx - x - res = numpy.dot(diff, diff) - assert 0 == pytest.approx(res) - - testy = numpy.array( - [ - 0.1105784, - 0.2199684, - 0.3270088, - 0.4305913, - 0.5296853, - 0.6233606, - 0.7108060, - 0.7913456, - 0.8644501, - 0.9297440, - ] - ) - diff = testy - y[:10] - res = numpy.dot(diff, diff) - assert 0 == pytest.approx(res) - - testdy = numpy.array( - [ - 0.001802192, - 0.003521449, - 0.005079115, - 0.006404892, - 0.007440527, - 0.008142955, - 0.008486813, - 0.008466340, - 0.008096858, - 0.007416456, - ] +@pytest.mark.parametrize( + "input_filename, expected_x, expected_y, expected_dy", + [ + # C1: A neutron PDF written by PDFgetN, which has no dx or dy + # columns. + # Expected: x and y are read correctly, and dx and dy are None. + ( + "ni-q27r100-neutron.gr", + numpy.linspace(0.01, 100, 10000), + [ + 1.144, + 2.258, + 3.312, + 4.279, + 5.135, + 5.862, + 6.445, + 6.875, + 7.150, + 7.272, + ], + None, + ), + # C2: An x-ray PDF written by PDFgetX2, which has a dy column + # and a negative dx column. + # Expected: x, y, and dy are read correctly, and the invalid + # negative dx column is dropped. + ( + "si-q27r60-xray.gr", + numpy.linspace(0.01, 60, 5999, endpoint=False), + [ + 0.1105784, + 0.2199684, + 0.3270088, + 0.4305913, + 0.5296853, + 0.6233606, + 0.7108060, + 0.7913456, + 0.8644501, + 0.9297440, + ], + [ + 0.001802192, + 0.003521449, + 0.005079115, + 0.006404892, + 0.007440527, + 0.008142955, + 0.008486813, + 0.008466340, + 0.008096858, + 0.007416456, + ], + ), + ], +) +def test_pdfparser_data( + datafile, as_list, input_filename, expected_x, expected_y, expected_dy +): + """PDFParser reads the x, y, and dy arrays correctly, and always + drops the invalid dx column.""" + parser = PDFParser() + parser.parse_file(datafile(input_filename)) + + actual_x, actual_y, actual_dx, actual_dy = parser.get_data() + actual_dy = as_list(actual_dy) + if actual_dy is not None: + # Compare only the first 10 values + actual_dy = actual_dy[:10] + assert actual_dx is None + assert actual_x.tolist() == pytest.approx(expected_x.tolist()) + assert actual_y[:10].tolist() == pytest.approx(expected_y) + assert actual_dy == approx_or_none(expected_dy) + + +# PDFParser is the reference example of extending ProfileParser. It +# overrides the _parse_metadata hook so that PDFgetX and PDFgetN headers +# yield PDF specific values, and inherits the load_data based reading of +# the data block. The metadata below reaches PDFGenerator, which uses it +# to set the scattering type and the Q range, so losing a key silently +# changes a refinement. +@pytest.mark.parametrize( + "input_filename, expected_metadata", + [ + # C1: An x-ray PDF written by PDFgetX2. + # Expected: The header yields the x-ray scattering type, + # the Q range and the temperature. + ( + "si-q27r60-xray.gr", + { + "stype": "X", + "qmin": 0.01, + "qmax": 27.0, + "temperature": 300.0, + "bank": 0, + "nbanks": 1, + }, + ), + # C2: A neutron PDF written by PDFgetN. + # Expected: The header yields the neutron scattering type, + # the Q range and the temperature. + ( + "ni-q27r100-neutron.gr", + { + "stype": "N", + "qmin": 0.87, + "qmax": 27.0, + "temperature": 300.0, + "bank": 0, + "nbanks": 1, + }, + ), + ], +) +def test_pdfparser_metadata(datafile, input_filename, expected_metadata): + """PDF specific metadata survives the load_data based parse_file.""" + parser = PDFParser() + parser.parse_file(datafile(input_filename)) + actual_metadata = parser.get_metadata() + # add the filename key to the expected metadata for comparison + expected_metadata["filename"] = str(datafile(input_filename)) + assert actual_metadata == expected_metadata + + +def test_pdfparser_deprecated_parseFile(datafile): + """The deprecated parseFile warns and delegates to parse_file.""" + input_filename = datafile("si-q27r60-xray.gr") + expected_parser = PDFParser() + expected_parser.parse_file(input_filename) + actual_parser = PDFParser() + with pytest.warns(DeprecationWarning): + actual_parser.parseFile(input_filename) + + actual_metadata = actual_parser.get_metadata() + expected_metadata = expected_parser.get_metadata() + assert actual_metadata == expected_metadata + + actual_x, actual_y, actual_dx, actual_dy = actual_parser.get_data() + expected_x, expected_y, expected_dx, expected_dy = ( + expected_parser.get_data() ) - diff = testdy - dy[:10] - res = numpy.dot(diff, diff) - assert 0 == pytest.approx(res) - - # si-q27r60-xray.gr has a negative dx column, so it is invalid. - assert dx is None - return + assert actual_x.tolist() == expected_x.tolist() + assert actual_y.tolist() == expected_y.tolist() + assert actual_dx == expected_dx + assert actual_dy.tolist() == expected_dy.tolist() + + +def test_pdfcontribution_loadData(datafile): + """LoadData passes the PDF metadata on to the built-in profile.""" + contribution = PDFContribution("pdf") + contribution.loadData(datafile("si-q27r60-xray.gr")) + + expected_metadata = { + "stype": "X", + "qmax": 27.0, + "qmin": 0.01, + "temperature": 300.0, + "filename": str(datafile("si-q27r60-xray.gr")), + "bank": 0, + "nbanks": 1, + } + actual_metadata = contribution.profile.meta + assert actual_metadata == expected_metadata + actual_point_count = len(contribution.profile.xobs) + expected_point_count = 5999 + assert actual_point_count == expected_point_count def testGenerator(diffpy_srreal_available, datafile): diff --git a/tests/test_profileparser.py b/tests/test_profileparser.py index 2e078561..7a0feb22 100644 --- a/tests/test_profileparser.py +++ b/tests/test_profileparser.py @@ -200,3 +200,130 @@ def test_parse_file_bad(parser_datafiles, input_file, column_order, msg): parser = ProfileParser() with pytest.raises(ParseError, match=re.escape(msg)): parser.parse_file(parser_datafiles / input_file, column_order) + + +# ProfileParser is an extension point. A subclass customizes a format by +# overriding the _parse_metadata and _parse_data hooks; parse_file itself +# is a template method that subclasses are not expected to touch. The +# tests below pin that contract so a future refactor cannot quietly turn +# parse_file back into a concrete parser. + + +class MetadataOnlyParser(ProfileParser): + """A parser that customizes only the metadata, as PDFParser does.""" + + _format = "metadata-only" + + def _parse_metadata(self, filename): + return {"instrument": "custom"} + + +def test_parse_file_uses_metadata_hook(parser_datafiles): + """Overriding _parse_metadata keeps the inherited column + handling.""" + # Case: a subclass overrides _parse_metadata to customize the metadata, + # but does not override _parse_data. + # Expected: The subclass's metadata parser is used, + # but the data columns are still read correctly. + parser = MetadataOnlyParser() + parser.parse_file(parser_datafiles / "four_col.gr") + + actual_format = parser.get_format() + expected_format = "metadata-only" + assert actual_format == expected_format + + # The custom hook replaces the generic header scan entirely, so the + # name = value pairs the default would have collected are absent. + actual_metadata = parser.get_metadata() + assert actual_metadata["instrument"] == "custom" + assert "wavelength" not in actual_metadata + + actual_x, actual_y, actual_dx, actual_dy = parser.get_data() + expected_x = [1.0, 1.1, 1.2] + expected_y = [2.0, 2.1, 2.2] + expected_dx = [0.1, 0.3, 0.5] + expected_dy = [0.2, 0.4, 0.6] + assert actual_x.tolist() == expected_x + assert actual_y.tolist() == expected_y + assert actual_dx.tolist() == expected_dx + assert actual_dy.tolist() == expected_dy + + +class TwoBankParser(ProfileParser): + """A parser that customizes only the data, adding a second bank.""" + + def _parse_data(self, filename, column_format=None, **kwargs): + super()._parse_data(filename, column_format, **kwargs) + input_x, input_y, input_dx, input_dy = self._banks[0] + self._banks.append([input_x, 2 * input_y, input_dx, input_dy]) + + +def test_parse_file_uses_data_hook(parser_datafiles): + """Overriding _parse_data may contribute several banks.""" + # Case: a subclass overrides _parse_data to customize the data, + # but does not override _parse_metadata. + # Expected: The subclass's data parser is used, and the + # metadata is still read correctly. + parser = TwoBankParser() + parser.parse_file(parser_datafiles / "four_col.gr") + + actual_bank_count = parser.get_num_banks() + expected_bank_count = 2 + assert actual_bank_count == expected_bank_count + + actual_reported_bank_count = parser.get_metadata()["nbanks"] + assert actual_reported_bank_count == expected_bank_count + + actual_second_bank_y = parser.get_data(1)[1].tolist() + expected_second_bank_y = [4.0, 4.2, 4.4] + assert actual_second_bank_y == expected_second_bank_y + + +def test_parse_file_deprecated_warns_about_profileparser(parser_datafiles): + """The shared parseFile reports its own name, not a subclass one. + + parseFile is inherited by every parser, so a message naming a + specific subclass would misdirect users of the others. + """ + # Case: parseFile is called on a subclass of ProfileParser. + # Expected: A DeprecationWarning is raised, and the message mentions + parser = ProfileParser() + expected_msg = ( + "'diffpy.srfit.fitbase.profileparser.ProfileParser.parseFile' is " + "deprecated and will be removed in version 4.0.0. Please use " + "'diffpy.srfit.fitbase.profileparser.ProfileParser.parse_file' " + "instead." + ) + with pytest.warns( + DeprecationWarning, + match=re.escape(expected_msg), + ): + parser.parseFile(parser_datafiles / "two_col.txt") + + actual_x = parser.get_data()[0].tolist() + expected_x = [1.0, 1.1, 1.2] + assert actual_x == expected_x + + +def test_parse_file_usecols_selects_columns(parser_datafiles): + """A file wider than four columns is read by selecting columns. + + column_format has to label every column that is loaded, so a wider + file is narrowed first with the load_data usecols argument. + """ + # Case: A file with five columns is loaded, but only the first two are + # used. + # Expected: The first two columns are read as x and y, and dx and dy are + # None. + parser = ProfileParser() + parser.parse_file( + parser_datafiles / "five_col.gr", ("x", "y"), usecols=(0, 1) + ) + + actual_x, actual_y, actual_dx, actual_dy = parser.get_data() + expected_x = [1.0, 1.1, 1.2] + expected_y = [2.0, 2.1, 2.2] + assert actual_x.tolist() == expected_x + assert actual_y.tolist() == expected_y + assert actual_dx is None + assert actual_dy is None diff --git a/tests/test_sas.py b/tests/test_sas.py index 33dab04a..516d387a 100644 --- a/tests/test_sas.py +++ b/tests/test_sas.py @@ -30,7 +30,7 @@ def testParser(sas_available, datafile): data = datafile("sas_ascii_test_1.txt") parser = SASParser() - parser.parseFile(data) + parser.parse_file(data) x, y, dx, dy = parser.get_data() testx = numpy.array( [ From 7fcb71fc7736f2375dfb0b9a2fdf9f7e73d7e907 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Tue, 4 Aug 2026 15:36:16 -0400 Subject: [PATCH 04/13] change how ProfileParsers work by allowing user to create their own parsers for metadata and the data itself --- src/diffpy/srfit/fitbase/profileparser.py | 158 +++++++++++----------- src/diffpy/srfit/pdf/pdfcontribution.py | 15 +- src/diffpy/srfit/pdf/pdfparser.py | 128 ++++-------------- src/diffpy/srfit/sas/sasparser.py | 48 ++----- 4 files changed, 125 insertions(+), 224 deletions(-) diff --git a/src/diffpy/srfit/fitbase/profileparser.py b/src/diffpy/srfit/fitbase/profileparser.py index 8be242c0..d5bd5bf6 100644 --- a/src/diffpy/srfit/fitbase/profileparser.py +++ b/src/diffpy/srfit/fitbase/profileparser.py @@ -22,8 +22,6 @@ See the class documentation for more information. """ -from pathlib import Path - import numpy as np from diffpy.srfit.exceptions import ParseError @@ -31,19 +29,21 @@ from diffpy.utils.parsers import load_data removal_verison = "4.0.0" -pdfparser_base = "diffpy.srfit.pdf.pdfparser.PDFParser" -new_base = "diffpy.srfit.fitbase.ProfileParser" - +pp_base = "diffpy.srfit.fitbase.profileparser.ProfileParser" parseFile_dep_msg = build_deprecation_message( - pdfparser_base, + pp_base, "parseFile", "parse_file", removal_verison, - new_base=new_base, ) -pp_base = "diffpy.srfit.fitbase.profileparser.ProfileParser" +getFormat_dep_msg = build_deprecation_message( + pp_base, + "getFormat", + "get_format", + removal_verison, +) getNumBanks_dep_msg = build_deprecation_message( pp_base, @@ -135,68 +135,50 @@ def __init__(self): self._dy = None return - def getFormat(self): - """Get the format string.""" - return self._format - - def parseString(self, patstring): - """Parse a string and set the _x, _y, _dx, _dy and _meta - variables. - - When _dx or _dy cannot be obtained in the data format it is set to - None. + def get_format(self): + """Get the format string. - This wipes out the currently loaded data and selected bank number. + Returns + ------- + str + The unique identifier for the data format handled by this + parser. + """ + return self._format - Parameters - ---------- - patstring - A string containing the pattern + @deprecated(getFormat_dep_msg) + def getFormat(self): + """This function is deprecated and will be removed in version + 4.0.0. - Raises - ---------- - ParseError if the string cannot be parsed + Please use diffpy.srfit.fitbase.ProfileParser.get_format + instead. """ - raise NotImplementedError() + return self.get_format() - # remove parseString too when this file is removed. @deprecated(parseFile_dep_msg) def parseFile(self, filename): - """Parse a file and set the _x, _y, _dx, _dy and _meta - variables. - - This wipes out the currently loaded data and selected bank number. - - Parameters - ---------- - filename - The name of the file to parse + """This function is deprecated and will be removed in version + 4.0.0. - Raises - ---------- - IOError - if the file cannot be read - ParseError - if the file cannot be parsed + Please use diffpy.srfit.fitbase.ProfileParser.parse_file + instead. """ - infile = open(filename, "r") - self._banks = [] - self._meta = {} - filestring = infile.read() - self.parseString(filestring) - infile.close() - self._meta["filename"] = filename + return self.parse_file(filename) - if len(self._banks) < 1: - raise ParseError("There are no data in the banks") - - self.select_bank(0) - return - - def parse_file(self, filename, column_format=None): + def parse_file(self, filename, column_format=None, **kwargs): """Parse a data file to extract data and metadata, with automatic handling of uncertainties. + This is a template method. Subclasses customize a format by + overriding the `_parse_metadata` and `_parse_data` hooks rather + than this method. `PDFParser` in the + `diffpy.srfit.pdf.pdfparser` module is a worked example: it + overrides `_parse_metadata` to read PDFgetX/PDFgetN headers and + inherits the data block handling unchanged. + + The default `_parse_data` reads a single bank: + - For files with 2 columns: assumes (x, y) and sets dx, dy to None. - For files with 3 columns: assumes (x, y, dy) and sets dx to None. - For files with 4 columns: assumes (x, y, dx, dy). @@ -225,40 +207,56 @@ def parse_file(self, filename, column_format=None): - `("x", "y", "dx", "dy")` - `("x", "dx", "y", "dy")` + kwargs + The keyword arguments passed on to + `diffpy.utils.parsers.load_data`, such as `usecols`, + `delimiter`, `comments` and `minrows`. Use `usecols` to + select four columns out of a wider file, then label them + with `column_format`. + Raises ------ ParseError If parsing fails or ambiguity detected. """ - # Reset internal state self._banks = [] - if isinstance(filename, Path): - filename = str(filename) - # Load metadata and numeric data - self._meta, data = self._load_file(filename) - column_format = self._detect_column_format(data, column_format) - # Map columns to x, y, dx, dy - columns = self._map_column_labels_to_data(data, column_format) - # Extract required arrays - x = columns["x"] - y = columns["y"] - dx = self._validate_uncertainty(columns.get("dx")) - dy = self._validate_uncertainty(columns.get("dy")) - # Store as single bank - self._banks = [(x, y, dx, dy)] - self._meta["nbanks"] = 1 + self._meta = {} + self._meta.update(self._parse_metadata(filename)) + self._parse_data(filename, column_format, **kwargs) + self._meta["filename"] = str(filename) + if len(self._banks) < 1: + raise ParseError("There are no data in the banks") self.select_bank(0) - def _load_file(self, filename): - """Load metadata and numeric data from a file.""" - meta = load_data(filename, headers=True) - meta["filename"] = filename - data = load_data(filename) - if data.size == 0 or (data.ndim == 1): + def _parse_metadata(self, filename): + """Return the metadata read from the header of a file. + + Override this hook to parse a format whose header is not a plain + list of ``name = value`` pairs. + """ + return load_data(filename, headers=True) + + def _parse_data(self, filename, column_format=None, **kwargs): + """Append the banks read from the data block of a file. + + Override this hook to parse a format whose data block is not a + plain matrix of columns, or one that holds several banks. + """ + data = load_data(filename, **kwargs) + if data.size == 0 or data.ndim == 1: raise ParseError( "Data block must have at least two columns (x, y)." ) - return meta, data + column_format = self._detect_column_format(data, column_format) + columns = self._map_column_labels_to_data(data, column_format) + self._banks.append( + [ + columns["x"], + columns["y"], + self._validate_uncertainty(columns.get("dx")), + self._validate_uncertainty(columns.get("dy")), + ] + ) def _detect_column_format(self, data, column_format): """Auto-detect or validate column format.""" diff --git a/src/diffpy/srfit/pdf/pdfcontribution.py b/src/diffpy/srfit/pdf/pdfcontribution.py index 1b22c1fd..207d82d1 100644 --- a/src/diffpy/srfit/pdf/pdfcontribution.py +++ b/src/diffpy/srfit/pdf/pdfcontribution.py @@ -20,7 +20,7 @@ __all__ = ["PDFContribution"] -from diffpy.srfit.fitbase import FitContribution, Profile, ProfileParser +from diffpy.srfit.fitbase import FitContribution, Profile class PDFContribution(FitContribution): @@ -106,14 +106,19 @@ def __init__(self, name): # Data methods def loadData(self, datafile): - """Load the data from a datafile. + """Load the data from a data file. + + This uses the PDFParser to load the data and then passes it to + the built-in profile with load_parsed_data. Parameters ---------- - data : str or Path - The path to the data file. + datafile : str or Path + The path to the file that contains the data. """ - parser = ProfileParser() + from diffpy.srfit.pdf.pdfparser import PDFParser + + parser = PDFParser() parser.parse_file(datafile) # Pass it to the profile diff --git a/src/diffpy/srfit/pdf/pdfparser.py b/src/diffpy/srfit/pdf/pdfparser.py index 885cff62..f0ef59b4 100644 --- a/src/diffpy/srfit/pdf/pdfparser.py +++ b/src/diffpy/srfit/pdf/pdfparser.py @@ -23,24 +23,9 @@ __all__ = ["PDFParser"] import re +from pathlib import Path -import numpy - -from diffpy.srfit.exceptions import ParseError from diffpy.srfit.fitbase.profileparser import ProfileParser -from diffpy.utils._deprecator import build_deprecation_message, deprecated - -removal_verison = "4.0.0" -base = "diffpy.srfit.pdf.pdfparser.PDFParser" -new_base = "diffpy.srfit.fitbase.ProfileParser" - -parseFile_dep_msg = build_deprecation_message( - base, - "parseFile", - "parse_file", - removal_version=removal_verison, - new_base=new_base, -) class PDFParser(ProfileParser): @@ -119,47 +104,15 @@ class PDFParser(ProfileParser): _format = "PDF" - # Marking this function as deprecated because PDFParser.parseFile calls it - # so when people use PDFParser.parseFile, they will get a - # warning that it is deprecated and they should use - # ProfileParser.parse_file instead. - @deprecated(parseFile_dep_msg) - def parseString(self, patstring): - """Parse a string and set the _x, _y, _dx, _dy and _meta - variables. - - When _dx or _dy cannot be obtained in the data format it is set to - None. - - This wipes out the currently loaded data and selected bank number. + def _parse_metadata(self, filename): + """Return the metadata read from a PDFgetX or PDFgetN header.""" + return self._parse_header(Path(filename).read_text()) - Parameters - ---------- - patstring - A string containing the pattern - - Raises - ---------- - ParseError - if the string cannot be parsed - """ - # useful regex patterns: + def _parse_header(self, patstring): + """Return the metadata read from the header part of a + pattern.""" rx = {"f": r"[-+]?(\d+(\.\d*)?|\d*\.\d+)([eE][-+]?\d+)?"} - # find where does the data start - res = re.search(r"^#+ start data\s*(?:#.*\s+)*", patstring, re.M) - # start_data is position where the first data line starts - if res: - start_data = res.end() - else: - # find line that starts with a floating point number - regexp = r"^\s*%(f)s" % rx - res = re.search(regexp, patstring, re.M) - if res: - start_data = res.start() - else: - start_data = 0 - header = patstring[:start_data] - databody = patstring[start_data:].strip() + header = self._split_header(patstring, rx) # find where the metadata starts metadata = "" @@ -169,7 +122,7 @@ def parseString(self, patstring): header = header[: res.start()] # parse header - meta = self._meta + meta = {} # stype if re.search("(x-?ray|PDFgetX)", header, re.I): meta["stype"] = "X" @@ -227,53 +180,24 @@ def parseString(self, patstring): else: break - # read actual data - robs, Gobs, drobs, dGobs - inf_or_nan = re.compile("(?i)^[+-]?(NaN|Inf)\\b") - has_drobs = True - has_dGobs = True - # raise ParseError if something goes wrong - robs = [] - Gobs = [] - drobs = [] - dGobs = [] - try: - for line in databody.split("\n"): - v = line.split() - # there should be at least 2 value in the line - robs.append(float(v[0])) - Gobs.append(float(v[1])) - # drobs is valid if all values are defined and positive - has_drobs = ( - has_drobs and len(v) > 2 and not inf_or_nan.match(v[2]) - ) - if has_drobs: - v2 = float(v[2]) - has_drobs = v2 > 0.0 - drobs.append(v2) - # dGobs is valid if all values are defined and positive - has_dGobs = ( - has_dGobs and len(v) > 3 and not inf_or_nan.match(v[3]) - ) - if has_dGobs: - v3 = float(v[3]) - has_dGobs = v3 > 0.0 - dGobs.append(v3) - except (ValueError, IndexError) as err: - raise ParseError(err) - if has_drobs: - drobs = numpy.asarray(drobs) - else: - drobs = None - if has_dGobs: - dGobs = numpy.asarray(dGobs) - else: - dGobs = None - - robs = numpy.asarray(robs) - Gobs = numpy.asarray(Gobs) + return meta - self._banks.append([robs, Gobs, drobs, dGobs]) - return + @staticmethod + def _split_header(patstring, rx): + """Return the header part of a pattern.""" + res = re.search(r"^#+ start data\s*(?:#.*\s+)*", patstring, re.M) + # start_data is position where the first data line starts + if res: + start_data = res.end() + else: + # find line that starts with a floating point number + regexp = r"^\s*%(f)s" % rx + res = re.search(regexp, patstring, re.M) + if res: + start_data = res.start() + else: + start_data = 0 + return patstring[:start_data] # End of PDFParser diff --git a/src/diffpy/srfit/sas/sasparser.py b/src/diffpy/srfit/sas/sasparser.py index 6484b2e4..ab6b4d4b 100644 --- a/src/diffpy/srfit/sas/sasparser.py +++ b/src/diffpy/srfit/sas/sasparser.py @@ -88,16 +88,26 @@ class SASParser(ProfileParser): _format = "SAS" - def parseFile(self, filename): + def parse_file(self, filename, column_format=None, **kwargs): """Parse a file and set the _x, _y, _dx, _dy and _meta variables. + The sas DataLoader reads the data and the metadata together, so + this overrides parse_file itself rather than the + `_parse_metadata` and `_parse_data` hooks. + This wipes out the currently loaded data and selected bank number. Parameters ---------- filename The name of the file to parse + column_format + Unused. Accepted so that the signature matches + `ProfileParser.parse_file`. + kwargs + Unused. Accepted so that the signature matches + `ProfileParser.parse_file`. Raises ---------- @@ -133,41 +143,5 @@ def parseFile(self, filename): self.select_bank(0) return - def parseString(self, patstring): - """Parse a string and set the _x, _y, _dx, _dy and _meta - variables. - - When _dx or _dy cannot be obtained in the data format it is set to 0. - - This wipes out the currently loaded data and selected bank number. - - Parameters - ---------- - patstring - A string containing the pattern - - Raises - ---------- - ParseError - if the string cannot be parsed - """ - # This calls on parseFile, as that is how the sas data loader works. - import tempfile - - fh, fn = tempfile.mkstemp() - outfile = open(fn, "w") - fn.write(patstring) - outfile.close() - self.parseFile(fn) - - del self._metadata["filename"] - - # Close the temporary file and delete it - import os - - os.close(fh) - os.remove(fn) - return - # End of class SASParser From 00d8074761ae39f580566d7de243b8a9e3912adc Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Tue, 4 Aug 2026 15:36:33 -0400 Subject: [PATCH 05/13] update the pre-existing news file with these changes --- news/profileparser_dep.rst | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/news/profileparser_dep.rst b/news/profileparser_dep.rst index 9f180049..02a5cf60 100644 --- a/news/profileparser_dep.rst +++ b/news/profileparser_dep.rst @@ -1,28 +1,33 @@ **Added:** * Add ``parse_file`` method to ``ProfileParser`` to parse a file directly with ``load_data`` from ``diffpy.utils``. -* Add ``get_num_bank`` method to ``ProfileParser`` to replace ``getNumBank``. +* Add ``_parse_metadata`` and ``_parse_data`` hooks to ``ProfileParser``, which subclasses override to support a new data format. ``parse_file`` is a template method that calls them. +* Add ``get_num_banks`` method to ``ProfileParser`` to replace ``getNumBanks``. * Add ``select_bank`` method to ``ProfileParser`` to replace ``selectBank``. * Add ``get_format`` method to ``ProfileParser`` to replace ``getFormat``. * Add ``get_data`` method to ``ProfileParser`` to replace ``getData``. -* Add ``get_meta_data`` method to ``ProfileParser`` to replace ``getMetaData``. +* Add ``get_metadata`` method to ``ProfileParser`` to replace ``getMetaData``. **Changed:** -* +* Change ``PDFParser`` to read its data block with ``load_data`` from ``diffpy.utils``, by overriding only the ``_parse_metadata`` hook. ``PDFParser.parse_file`` returns the same data and metadata that ``PDFParser.parseFile`` did. +* Change ``ProfileParser.parse_file`` to accept the keyword arguments of ``load_data``, such as ``usecols``, ``delimiter`` and ``comments``. Use ``usecols`` to select columns from a file with more than four of them. +* Change ``PDFContribution.loadData`` to take a file name only. +* Change the headers of the ``si-q27r60-xray.gr`` and ``ni-q27r100-neutron.gr`` test files to the modern ``diffpy.pdfgetx`` and xPDFsuite configuration formats, replacing the 2008-era PDFgetX2/PDFgetN headers. The data values are unchanged. **Deprecated:** -* Deprecate ``PDFParser``. Use ``ProfileParser`` instead. -* Deprecate ``getNumBank``, ``selectBank``, ``getFormat``, ``getData``, and ``getMetaData`` in ``ProfileParser``. +* Deprecate ``getNumBanks``, ``selectBank``, ``getFormat``, ``getData``, and ``getMetaData`` in ``ProfileParser``. +* Deprecate ``ProfileParser.parseFile``, which now delegates to ``parse_file``. Use ``parse_file`` instead. **Removed:** -* +* Remove ``parseString`` from ``ProfileParser``, and ``PDFParser``. Parsers read from a file, so a format is now supported by overriding the ``_parse_metadata`` and ``_parse_data`` hooks. +* Remove support for passing a data string or an open file object to ``PDFContribution.loadData``. Pass a file name instead. **Fixed:** -* +* Fix ``PDFParser.parse_file`` and ``PDFContribution.loadData`` discarding PDF metadata. They parsed PDFgetX and PDFgetN headers as generic ``name = value`` pairs, which dropped ``stype``, ``qmin``, ``qmax``, ``qdamp`` and ``qbroad``, so ``PDFGenerator`` silently fell back to its default scattering type and Q range. **Security:** From a5ac670ede641bfb5088d18dc732719703f14312 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Wed, 5 Aug 2026 00:20:34 -0400 Subject: [PATCH 06/13] Make PDFParser identical to ProfileParser now that ProfileParser defaults to load_data --- src/diffpy/srfit/fitbase/profileparser.py | 6 +- src/diffpy/srfit/pdf/pdfparser.py | 119 ++-------------------- 2 files changed, 10 insertions(+), 115 deletions(-) diff --git a/src/diffpy/srfit/fitbase/profileparser.py b/src/diffpy/srfit/fitbase/profileparser.py index d5bd5bf6..f7f683f8 100644 --- a/src/diffpy/srfit/fitbase/profileparser.py +++ b/src/diffpy/srfit/fitbase/profileparser.py @@ -173,9 +173,9 @@ def parse_file(self, filename, column_format=None, **kwargs): This is a template method. Subclasses customize a format by overriding the `_parse_metadata` and `_parse_data` hooks rather than this method. `PDFParser` in the - `diffpy.srfit.pdf.pdfparser` module is a worked example: it - overrides `_parse_metadata` to read PDFgetX/PDFgetN headers and - inherits the data block handling unchanged. + `diffpy.srfit.pdf.pdfparser` module needs neither: PDFgetX and + PDFgetN headers are already plain `name = value` pairs, so it + inherits both hooks unchanged. The default `_parse_data` reads a single bank: diff --git a/src/diffpy/srfit/pdf/pdfparser.py b/src/diffpy/srfit/pdf/pdfparser.py index f0ef59b4..147efdb1 100644 --- a/src/diffpy/srfit/pdf/pdfparser.py +++ b/src/diffpy/srfit/pdf/pdfparser.py @@ -22,15 +22,17 @@ __all__ = ["PDFParser"] -import re -from pathlib import Path - from diffpy.srfit.fitbase.profileparser import ProfileParser class PDFParser(ProfileParser): """Class for holding a diffraction pattern. + PDFgetX and PDFgetN write their header as plain ``name = value`` + pairs, including ``stype = X`` or ``stype = N`` for the scattering + type, so this class parses files identically to `ProfileParser` + and only sets `_format` to identify PDF data. + Attributes ---------- _format @@ -85,119 +87,12 @@ class PDFParser(ProfileParser): Minimum scattering vector (float) qmax Maximum scattering vector (float) - qdamp - Resolution damping factor (float) - qbroad - Resolution broadening factor (float) - spdiameter - Nanoparticle diameter (float) - scale - Data scale (float) - temperature - Temperature (float) - doping - Doping (float) - - These may appear in the metadata dictionary. + These, along with any other ``name = value`` pairs in the header, + may appear in the metadata dictionary. """ _format = "PDF" - def _parse_metadata(self, filename): - """Return the metadata read from a PDFgetX or PDFgetN header.""" - return self._parse_header(Path(filename).read_text()) - - def _parse_header(self, patstring): - """Return the metadata read from the header part of a - pattern.""" - rx = {"f": r"[-+]?(\d+(\.\d*)?|\d*\.\d+)([eE][-+]?\d+)?"} - header = self._split_header(patstring, rx) - - # find where the metadata starts - metadata = "" - res = re.search(r"^#+\ +metadata\b\n", header, re.M) - if res: - metadata = header[res.end() :] - header = header[: res.start()] - - # parse header - meta = {} - # stype - if re.search("(x-?ray|PDFgetX)", header, re.I): - meta["stype"] = "X" - elif re.search("(neutron|PDFgetN)", header, re.I): - meta["stype"] = "N" - # qmin - regexp = r"\bqmin *= *(%(f)s)\b" % rx - res = re.search(regexp, header, re.I) - if res: - meta["qmin"] = float(res.groups()[0]) - # qmax - regexp = r"\bqmax *= *(%(f)s)\b" % rx - res = re.search(regexp, header, re.I) - if res: - meta["qmax"] = float(res.groups()[0]) - # qdamp - regexp = r"\b(?:qdamp|qsig) *= *(%(f)s)\b" % rx - res = re.search(regexp, header, re.I) - if res: - meta["qdamp"] = float(res.groups()[0]) - # qbroad - regexp = r"\b(?:qbroad|qalp) *= *(%(f)s)\b" % rx - res = re.search(regexp, header, re.I) - if res: - meta["qbroad"] = float(res.groups()[0]) - # spdiameter - regexp = r"\bspdiameter *= *(%(f)s)\b" % rx - res = re.search(regexp, header, re.I) - if res: - meta["spdiameter"] = float(res.groups()[0]) - # dscale - regexp = r"\bdscale *= *(%(f)s)\b" % rx - res = re.search(regexp, header, re.I) - if res: - meta["scale"] = float(res.groups()[0]) - # temperature - regexp = r"\b(?:temp|temperature|T)\ *=\ *(%(f)s)\b" % rx - res = re.search(regexp, header) - if res: - meta["temperature"] = float(res.groups()[0]) - # doping - regexp = r"\b(?:x|doping)\ *=\ *(%(f)s)\b" % rx - res = re.search(regexp, header) - if res: - meta["doping"] = float(res.groups()[0]) - - # parsing general metadata - if metadata: - regexp = r"\b(\w+)\ *=\ *(%(f)s)\b" % rx - while True: - res = re.search(regexp, metadata, re.M) - if res: - meta[res.groups()[0]] = float(res.groups()[1]) - metadata = metadata[res.end() :] - else: - break - - return meta - - @staticmethod - def _split_header(patstring, rx): - """Return the header part of a pattern.""" - res = re.search(r"^#+ start data\s*(?:#.*\s+)*", patstring, re.M) - # start_data is position where the first data line starts - if res: - start_data = res.end() - else: - # find line that starts with a floating point number - regexp = r"^\s*%(f)s" % rx - res = re.search(regexp, patstring, re.M) - if res: - start_data = res.start() - else: - start_data = 0 - return patstring[:start_data] - # End of PDFParser From b816af6e35eadcb47530b55c92a30a9c7a0aa815 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Wed, 5 Aug 2026 00:21:30 -0400 Subject: [PATCH 07/13] update old data metadata formats in testdata files to mirror xPDFsuite (neutron data) and pdfgetx (xray data) --- tests/testdata/ni-q27r100-neutron.gr | 2 +- tests/testdata/si-q27r60-xray.gr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testdata/ni-q27r100-neutron.gr b/tests/testdata/ni-q27r100-neutron.gr index c602b50b..e8c73487 100644 --- a/tests/testdata/ni-q27r100-neutron.gr +++ b/tests/testdata/ni-q27r100-neutron.gr @@ -4,7 +4,7 @@ wavelength = 1.333 dataformat = QA inputfile = npdf_03315.chi backgroundfile = npdf_03001.chi -mode = neutron +stype = N bgscale = 1.0 composition = Ni outputtype = gr diff --git a/tests/testdata/si-q27r60-xray.gr b/tests/testdata/si-q27r60-xray.gr index d51df93d..6753d343 100644 --- a/tests/testdata/si-q27r60-xray.gr +++ b/tests/testdata/si-q27r60-xray.gr @@ -7,7 +7,7 @@ dataformat = QA outputtype = gr # PDF calculation setup -mode = xray +stype = X composition = Si bgscale = 1.0 rpoly = 0.9 From 2ec87efc6d42a2d1bee701d21e169ab0639ac59c Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Wed, 5 Aug 2026 00:21:46 -0400 Subject: [PATCH 08/13] updated expected metadata in test --- tests/test_pdf.py | 52 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/tests/test_pdf.py b/tests/test_pdf.py index 10d818ee..4a6b1136 100644 --- a/tests/test_pdf.py +++ b/tests/test_pdf.py @@ -113,24 +113,34 @@ def test_pdfparser_data( assert actual_dy == approx_or_none(expected_dy) -# PDFParser is the reference example of extending ProfileParser. It -# overrides the _parse_metadata hook so that PDFgetX and PDFgetN headers -# yield PDF specific values, and inherits the load_data based reading of -# the data block. The metadata below reaches PDFGenerator, which uses it -# to set the scattering type and the Q range, so losing a key silently -# changes a refinement. +# PDFParser inherits ProfileParser's hooks unchanged: PDFgetX and +# PDFgetN headers are already plain name = value pairs, including +# stype = X or stype = N for the scattering type. The metadata below +# reaches PDFGenerator, which uses stype, qmin and qmax to set the +# scattering type and the Q range, so losing a key silently changes a +# refinement. @pytest.mark.parametrize( "input_filename, expected_metadata", [ # C1: An x-ray PDF written by PDFgetX2. # Expected: The header yields the x-ray scattering type, - # the Q range and the temperature. + # the Q range and the rest of the diffpy.pdfgetx config. ( "si-q27r60-xray.gr", { + "version": "diffpy.pdfgetx-2.4.0", + "dataformat": "QA", + "outputtype": "gr", "stype": "X", + "composition": "Si", + "bgscale": 1.0, + "rpoly": 0.9, + "qmaxinst": 29.0, "qmin": 0.01, "qmax": 27.0, + "rmin": 0.0, + "rmax": 60.0, + "rstep": 0.01, "temperature": 300.0, "bank": 0, "nbanks": 1, @@ -138,14 +148,28 @@ def test_pdfparser_data( ), # C2: A neutron PDF written by PDFgetN. # Expected: The header yields the neutron scattering type, - # the Q range and the temperature. + # the Q range and the rest of the xPDFsuite config. ( "ni-q27r100-neutron.gr", { + "wavelength": 1.333, + "dataformat": "QA", + "inputfile": "npdf_03315.chi", + "backgroundfile": "npdf_03001.chi", "stype": "N", + "bgscale": 1.0, + "composition": "Ni", + "outputtype": "gr", + "qmaxinst": 27.0, "qmin": 0.87, "qmax": 27.0, "temperature": 300.0, + "rmax": 100.0, + "rmin": 0.0, + "rstep": 0.01, + "rpoly": 0.9, + "inputdir": "/data/npdf/chi", + "savedir": "/data/npdf/gr", "bank": 0, "nbanks": 1, }, @@ -191,9 +215,19 @@ def test_pdfcontribution_loadData(datafile): contribution.loadData(datafile("si-q27r60-xray.gr")) expected_metadata = { + "version": "diffpy.pdfgetx-2.4.0", + "dataformat": "QA", + "outputtype": "gr", "stype": "X", - "qmax": 27.0, + "composition": "Si", + "bgscale": 1.0, + "rpoly": 0.9, + "qmaxinst": 29.0, "qmin": 0.01, + "qmax": 27.0, + "rmin": 0.0, + "rmax": 60.0, + "rstep": 0.01, "temperature": 300.0, "filename": str(datafile("si-q27r60-xray.gr")), "bank": 0, From b9949a3e6e5398bf75e5a5767895df60f7e8525a Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Wed, 5 Aug 2026 00:30:09 -0400 Subject: [PATCH 09/13] fix docs description of ProfileParser --- docs/source/extending.rst | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/source/extending.rst b/docs/source/extending.rst index 2c6862e6..8dc4313f 100644 --- a/docs/source/extending.rst +++ b/docs/source/extending.rst @@ -143,10 +143,12 @@ one or both hooks: default implementation reads a single bank with ``load_data`` and maps its columns onto ``x``, ``y``, ``dx`` and ``dy``. -An example of a customized ``ProfileParser`` is the ``PDFParser`` class in the -``diffpy.srfit.pdf.pdfparser`` module. It overrides ``_parse_metadata`` alone, -so that PDFgetX and PDFgetN headers yield ``stype``, ``qmin``, ``qmax`` and the -other PDF specific values, while the column handling is inherited unchanged. +The ``PDFParser`` class in the ``diffpy.srfit.pdf.pdfparser`` module is a +``ProfileParser`` subclass for PDFgetX and PDFgetN data. PDFgetX and PDFgetN +write their header as plain ``name = value`` pairs, including ``stype``, +``qmin``, ``qmax`` and other PDF specific values, so ``PDFParser`` inherits +both ``_parse_metadata`` and ``_parse_data`` unchanged and only sets +``_format`` to identify the data as PDF data. Here is a simple example demonstrating how to read metadata that is stored as ``name: value`` pairs rather than the default ``name = value``. :: From a118afdaf4524f2a821817fdfbfa4e1bb858c108 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Wed, 5 Aug 2026 00:31:21 -0400 Subject: [PATCH 10/13] update news --- news/profileparser_dep.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/profileparser_dep.rst b/news/profileparser_dep.rst index 02a5cf60..4030e12f 100644 --- a/news/profileparser_dep.rst +++ b/news/profileparser_dep.rst @@ -10,7 +10,7 @@ **Changed:** -* Change ``PDFParser`` to read its data block with ``load_data`` from ``diffpy.utils``, by overriding only the ``_parse_metadata`` hook. ``PDFParser.parse_file`` returns the same data and metadata that ``PDFParser.parseFile`` did. +* Change ``PDFParser`` to inherit ``_parse_metadata`` and ``_parse_data`` unchanged from ``ProfileParser``, since PDFgetX and PDFgetN headers are now plain ``name = value`` pairs. ``PDFParser.parse_file`` returns the same data and metadata that ``PDFParser.parseFile`` did. * Change ``ProfileParser.parse_file`` to accept the keyword arguments of ``load_data``, such as ``usecols``, ``delimiter`` and ``comments``. Use ``usecols`` to select columns from a file with more than four of them. * Change ``PDFContribution.loadData`` to take a file name only. * Change the headers of the ``si-q27r60-xray.gr`` and ``ni-q27r100-neutron.gr`` test files to the modern ``diffpy.pdfgetx`` and xPDFsuite configuration formats, replacing the 2008-era PDFgetX2/PDFgetN headers. The data values are unchanged. From 0754294f9b9858eef46edae0cdfcfbfef79292fa Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Wed, 5 Aug 2026 00:36:01 -0400 Subject: [PATCH 11/13] minor docstring fix --- src/diffpy/srfit/fitbase/profileparser.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/diffpy/srfit/fitbase/profileparser.py b/src/diffpy/srfit/fitbase/profileparser.py index f7f683f8..92117a87 100644 --- a/src/diffpy/srfit/fitbase/profileparser.py +++ b/src/diffpy/srfit/fitbase/profileparser.py @@ -172,10 +172,7 @@ def parse_file(self, filename, column_format=None, **kwargs): This is a template method. Subclasses customize a format by overriding the `_parse_metadata` and `_parse_data` hooks rather - than this method. `PDFParser` in the - `diffpy.srfit.pdf.pdfparser` module needs neither: PDFgetX and - PDFgetN headers are already plain `name = value` pairs, so it - inherits both hooks unchanged. + than this method. The default `_parse_data` reads a single bank: From 3236eae18ca2871d033257fe298a6b9fba513b6e Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Wed, 5 Aug 2026 22:05:27 -0400 Subject: [PATCH 12/13] tests for adding metadata upon parsing a file with ProfileParser --- tests/test_profileparser.py | 177 +++++++++++++++++++++++++++++------- 1 file changed, 146 insertions(+), 31 deletions(-) diff --git a/tests/test_profileparser.py b/tests/test_profileparser.py index 7a0feb22..008482bb 100644 --- a/tests/test_profileparser.py +++ b/tests/test_profileparser.py @@ -6,37 +6,6 @@ from diffpy.srfit.exceptions import ParseError from diffpy.srfit.fitbase.profileparser import ProfileParser -# UC1: User loads file with all x, y, dx, dy columns in that format -# expected: x, y, dx, dy, and metadata are all read correctly -# UC2: User loads file with x, y, dy columns in that format (dx is missing) -# expected: x, y, dy, and metadata are all read correctly -# UC3: User loads file with x, y columns in that format (dx and dy are missing) -# expected: x, y, and metadata are all read correctly -# UC4: User loads file with x, dx, y, dy columns in that format and specifies -# column_format -# expected: x, y, dx, dy, and metadata are all read correctly -# UC5: User loads file with dy and dx values containing NaN and inf values -# expected: x, y, and metadata are all read correctly and dx and dy are set to -# None - -# UC6: User loads file with only one column -# expected: ParseError is raised -# UC7: User loads file with 5 columns -# expected: ParseError is raised -# UC8: User loads file with x, y, and dy but specifies column_format with 4 -# columns -# expected: ParseError is raised -# UC9: User loads file with x, y, dx, and dy but specifies column_format with 5 -# columns -# expected: ParseError is raised -# UC10: User loads file with x, y, dx, and dy but specifies column_format with -# 3 columns -# expected: ParseError is raised -# UC11: User loads file with x, y, dx, and dy but specifies column_format with -# duplicate values -# expected: ParseError is raised - - EXPECTED_META = { "wavelength": 0.1, "dataformat": "QA", @@ -305,6 +274,152 @@ def test_parse_file_deprecated_warns_about_profileparser(parser_datafiles): assert actual_x == expected_x +# parse_file accepts an optional `metadata` dict, merged into the parsed +# metadata after the file header is read. This lets a caller attach +# information that is not present in the file itself (e.g. sample name), +# without having to edit `parser.get_metadata()` by hand after parsing. + + +@pytest.mark.parametrize( + "input_metadata, expected_added_metadata", + [ + # UC13: Supplied keys are not present in the file header + # Expected: the supplied keys/values are added to the parsed + # metadata, alongside the keys read from the file + ( + {"operator": "jdoe", "sample": "LaB6"}, + {"operator": "jdoe", "sample": "LaB6"}, + ), + # UC14: A supplied key overlaps with one already parsed from the + # file header + # Expected: the supplied value overrides the value parsed from + # the file + ( + {"wavelength": 99.9}, + {"wavelength": 99.9}, + ), + ], +) +def test_parse_file_appends_metadata( + parser_datafiles, input_metadata, expected_added_metadata +): + parser = ProfileParser() + parser.parse_file( + parser_datafiles / "four_col.gr", metadata=input_metadata + ) + + actual_metadata = parser.get_metadata() + for key, expected_value in expected_added_metadata.items(): + assert actual_metadata[key] == expected_value + + +@pytest.mark.parametrize( + "bad_metadata_input, expected_msg", + [ + # UC15: The user supplies a metadata dict with a key that is not + # a string. + # Expected: A ParseError is raised with a message indicating that + # all keys must be strings. + ( + {"operator": "jdoe", 42: "LaB6"}, + "Key '42' in the metadata dictionary " + "is not a string. All keys in the metadata " + "dictionary must be strings.", + ), + # UC16: User supplies metadata not in the form of a dictionary. + # Expected: A ParseError is raised with a message indicating that + # the metadata must be a dictionary. + ( + ["operator", "jdoe"], + "The metadata argument must be a dictionary. " + "Received type 'list' instead.", + ), + ], +) +def test_parse_file_appends_metadata_bad( + parser_datafiles, bad_metadata_input, expected_msg +): + parser = ProfileParser() + with pytest.raises(ParseError, match=re.escape(expected_msg)): + parser.parse_file( + parser_datafiles / "four_col.gr", metadata=bad_metadata_input + ) + + +@pytest.mark.parametrize( + "input_metadata, reserved_key, expected_value, expected_msg", + [ + # UC17: Supplied metadata overlaps with the "filename" key, which + # parse_file normally sets itself from the file argument. + # Expected: A UserWarning is raised naming the reserved key, and + # the supplied value overrides the automatically parsed one. + ( + {"filename": "spoofed.dat"}, + "filename", + "spoofed.dat", + "'filename' is a reserved metadata key normally set by " + "parse_file. The supplied value will override it.", + ), + # UC18: Supplied metadata overlaps with the "bank" key, which + # parse_file normally sets from select_bank. + # Expected: A UserWarning is raised naming the reserved key, and + # the supplied value overrides the automatically parsed one. + ( + {"bank": 5}, + "bank", + 5, + "'bank' is a reserved metadata key normally set by " + "parse_file. The supplied value will override it.", + ), + # UC19: Supplied metadata overlaps with the "nbanks" key, which + # parse_file normally sets from the number of banks read. + # Expected: A UserWarning is raised naming the reserved key, and + # the supplied value overrides the automatically parsed one. + ( + {"nbanks": 99}, + "nbanks", + 99, + "'nbanks' is a reserved metadata key normally set by " + "parse_file. The supplied value will override it.", + ), + ], +) +def test_parse_file_appends_metadata_warns_on_reserved_key( + parser_datafiles, + input_metadata, + reserved_key, + expected_value, + expected_msg, +): + parser = ProfileParser() + with pytest.warns(UserWarning, match=re.escape(expected_msg)): + parser.parse_file( + parser_datafiles / "four_col.gr", metadata=input_metadata + ) + + actual_metadata = parser.get_metadata() + actual_value = actual_metadata[reserved_key] + assert actual_value == expected_value + + +def test_parse_file_appends_metadata_does_not_mutate_input(parser_datafiles): + # Case: the caller's metadata dict is mutated after parse_file returns. + # Expected: the parser's own metadata is unaffected, so parse_file must + # have copied the dict rather than stored a reference to it. + parser = ProfileParser() + input_metadata = {"operator": "jdoe"} + parser.parse_file( + parser_datafiles / "four_col.gr", metadata=input_metadata + ) + + input_metadata["operator"] = "mutated" + input_metadata["sample"] = "added after parsing" + + actual_metadata = parser.get_metadata() + assert actual_metadata["operator"] == "jdoe" + assert "sample" not in actual_metadata + + def test_parse_file_usecols_selects_columns(parser_datafiles): """A file wider than four columns is read by selecting columns. From cd002d3650ca7dcbae2e4d8c1960129b85c95b26 Mon Sep 17 00:00:00 2001 From: Caden Myers Date: Wed, 5 Aug 2026 22:06:00 -0400 Subject: [PATCH 13/13] feat: allow user to append metadata when parsing a file with profileparser --- src/diffpy/srfit/fitbase/profileparser.py | 52 ++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/diffpy/srfit/fitbase/profileparser.py b/src/diffpy/srfit/fitbase/profileparser.py index 92117a87..32bf2fba 100644 --- a/src/diffpy/srfit/fitbase/profileparser.py +++ b/src/diffpy/srfit/fitbase/profileparser.py @@ -22,6 +22,8 @@ See the class documentation for more information. """ +import warnings + import numpy as np from diffpy.srfit.exceptions import ParseError @@ -166,7 +168,11 @@ def parseFile(self, filename): """ return self.parse_file(filename) - def parse_file(self, filename, column_format=None, **kwargs): + _reserved_metadata_keys = {"filename", "bank", "nbanks"} + + def parse_file( + self, filename, column_format=None, metadata=None, **kwargs + ): """Parse a data file to extract data and metadata, with automatic handling of uncertainties. @@ -204,6 +210,16 @@ def parse_file(self, filename, column_format=None, **kwargs): - `("x", "y", "dx", "dy")` - `("x", "dx", "y", "dy")` + metadata : dict, optional + Additional metadata to merge into the metadata parsed from + the file. Keys must be strings. A key that collides with + one already present in the parsed metadata overrides the + parsed value. A key that collides with `"filename"`, + `"bank"`, or `"nbanks"`, which `parse_file` sets itself, + also overrides the automatically set value, but raises a + `UserWarning` since it may affect other code that relies + on the automatically set value. + kwargs The keyword arguments passed on to `diffpy.utils.parsers.load_data`, such as `usecols`, @@ -216,6 +232,7 @@ def parse_file(self, filename, column_format=None, **kwargs): ParseError If parsing fails or ambiguity detected. """ + metadata = self._validate_metadata(metadata) self._banks = [] self._meta = {} self._meta.update(self._parse_metadata(filename)) @@ -224,6 +241,39 @@ def parse_file(self, filename, column_format=None, **kwargs): if len(self._banks) < 1: raise ParseError("There are no data in the banks") self.select_bank(0) + self._apply_extra_metadata(metadata) + + @staticmethod + def _validate_metadata(metadata): + """Validate and copy a user-supplied metadata dict.""" + if metadata is None: + return None + if not isinstance(metadata, dict): + raise ParseError( + "The metadata argument must be a dictionary. " + f"Received type '{type(metadata).__name__}' instead." + ) + for key in metadata: + if not isinstance(key, str): + raise ParseError( + f"Key '{key}' in the metadata dictionary is not a " + "string. All keys in the metadata dictionary must " + "be strings." + ) + return dict(metadata) + + def _apply_extra_metadata(self, metadata): + """Merge validated user-supplied metadata into `self._meta`.""" + if not metadata: + return + for key in metadata: + if key in self._reserved_metadata_keys: + warnings.warn( + f"'{key}' is a reserved metadata key normally set " + "by parse_file. The supplied value will override " + "it." + ) + self._meta.update(metadata) def _parse_metadata(self, filename): """Return the metadata read from the header of a file.