Uncertainty propagation and error-models equivalence#

%matplotlib inline
# use `widget` instead of `inline` for better user-exeperience. `inline` allows to store plots into notebooks.
import time
start_time = time.perf_counter()
import sys
print(sys.executable)
print(sys.version)
import os
os.environ["PYOPENCL_COMPILER_OUTPUT"] = "0"
/users/kieffer/.venv/py313/bin/python3
3.13.1 | packaged by conda-forge | (main, Jan 13 2025, 09:53:10) [GCC 13.3.0]
pix = 100e-6
shape = (1024, 1024)
npt = 1000
nimg = 1000
wl = 1e-10
I0 = 1e2
kwargs = {"npt":npt,
         "correctSolidAngle":True,
         "polarization_factor":0.99,
         "safe":False,
         "error_model":"poisson",
         "method":("full", "csr", "opencl"),
         }
         # "normalization_factor": 1.0}
import numpy
from matplotlib.pyplot import subplots
from matplotlib.colors import LogNorm
import logging
logging.basicConfig(level=logging.ERROR)
import pyFAI
print(f"pyFAI version: {pyFAI.version}")
from pyFAI.detectors import Detector
from pyFAI.gui import jupyter
detector = Detector(pix, pix)
detector.shape = detector.max_shape = shape
print(detector)
flat = numpy.random.random(shape)*1+1
pyFAI version: 2026.9.0
Detector Detector	 PixelSize= 100µm, 100µm	 BottomRight (3)
ai_init = {"dist":1.0,
           "poni1":0.0,
           "poni2":0.0,
           "rot1":-0.05,
           "rot2":+0.05,
           "rot3":0.0,
           "detector":detector,
           "wavelength":wl}
ai = pyFAI.load(ai_init)
print(ai)
Detector Detector	 PixelSize= 100µm, 100µm	 BottomRight (3)
Wavelength= 1.000000 Å
SampleDetDist= 1.000000e+00 m	PONI= 0.000000e+00, 0.000000e+00 m	rot1=-0.050000  rot2=0.050000  rot3=0.000000 rad
DirectBeamDist= 1002.504 mm	Center: x=500.417, y=501.043 pix	Tilt= 4.051° tiltPlanRotation= 45.036° λ= 1.000Å
# Generation of a "SAXS-like" curve with the shape of a lorentzian curve
unit="q_nm^-1"
q = numpy.linspace(0, ai.array_from_unit(unit=unit).max(), npt)
I = I0/(1+q**2)
fig, ax = subplots()
ax.semilogy(q, I, label="Simulated signal")
ax.set_xlabel("q ($nm^{-1}$)")
ax.set_ylabel("I (count)")
ax.set_title("SAXS-like curve with good statistical characteristics")
ax.legend()
pass
../../../_images/55596e6a827bf25363281404f74a4b733dd34c52335a1f8d70f0835b72d59793.png
#Reconstruction of diffusion image:
img_theo = ai.calcfrom1d(q, I, dim1_unit="q_nm^-1",
                         correctSolidAngle=True,
                         polarization_factor=None,
                         flat=flat)
kwargs["flat"] = flat
img_poisson = numpy.random.poisson(img_theo)
fig, ax = subplots(1, 2)
ax[0].imshow(img_theo, norm=LogNorm())
_=ax[0].set_title("Diffusion image (theo)")
ax[1].imshow(img_poisson, norm=LogNorm())
_=ax[1].set_title("Diffusion image (noisy)")
../../../_images/46f8fa8f11784d20d72e369fc2f5313ee41e5aa62d9110ef63a53f8c429be3e1.png

Azimuthal Integration#

factor = 1e-6
alpha=0.6
k = kwargs.copy()
k["error_model"] = "azimuthal"
np = kwargs.copy()
np["method"] = ("full", "csr", "python")
npa = np.copy()
npa["error_model"] = "azimuthal"


ref = ai.integrate1d(img_poisson, **kwargs)

ref_np = ai.integrate1d(img_poisson, **np)
res_azim = ai.integrate1d(img_poisson, **k)
np_azim = ai.integrate1d(img_poisson, **npa)
res_renorm = ai.integrate1d(img_poisson, normalization_factor=factor, **kwargs)
res_azim_renorm = ai.integrate1d(img_poisson, normalization_factor=factor, **k)

ax = jupyter.plot1d(res_azim)
ax.plot(res_renorm.radial, res_renorm.sem*factor, "1",alpha=alpha,  label="sem poisson, scaled")
ax.plot(ref.radial, ref.sem, label="sem poisson unscale")
ax.plot(res_azim_renorm.radial, res_azim_renorm.sem*factor,"2", alpha=alpha, label="sem azimuthal, scaled")
ax.plot(res_azim.radial, res_azim.sem, alpha=alpha, label="sem azimuthal, unscaled")

ax.plot(res_renorm.radial, res_renorm.std*factor,"1", alpha=alpha, label="std poisson, scaled")
ax.plot(ref.radial, ref.std, label="std poisson unscale")
ax.plot(res_azim_renorm.radial, res_azim_renorm.std*factor, "2", alpha=alpha, label="std azimuthal, scaled")
ax.plot(res_azim.radial, res_azim.std, alpha=alpha,label="std azimuthal, unscaled")


ax.legend()
_=ax.set_title("Azimuthal integration")
../../../_images/c97c3191e5c546c0cbd62947e2ead4a86d668386e9c3691c14fc300a8f2e72db.png
fig,ax = subplots()
what = "sum_variance"
ax.plot(np_azim.__getattribute__(what), label="azimuth summed variance")
ax.plot(ref_np.__getattribute__(what), label="poisson summed variance")
ax.legend()
<matplotlib.legend.Legend at 0x7f2e18177750>
../../../_images/7c24b64e1753ae33a654a662b51a2a7c27f2a388c4f14da3a2da7a473e0293ac.png
fig,ax = subplots()
what = "sum_variance"
ax.plot(ref_np.__getattribute__(what), label="poisson numpy")
ax.plot(ref.__getattribute__(what),"1", alpha=0.5, label="poisson opencl")
ax.plot(np_azim.__getattribute__(what), alpha=0.5, label="azimuth numpy")
ax.plot(res_azim.__getattribute__(what), "2", alpha=0.5, label="azimuth opencl")
ax.legend()
<matplotlib.legend.Legend at 0x7f2e181f3ed0>
../../../_images/bf36d7fb964372ffb7acb72b31d5f27b8a5fbb32cca39c6d3375ac587173e186.png

Sigma clipping#

factor = 1e-6
kwargs["method"] = ("no", "csr", "opencl")
k = kwargs.copy()
k["error_model"] = "azimuthal"
npp = kwargs.copy()
npp["method"] = ("no", "csr", "python")
npa = npp.copy()
npa["error_model"] = "azimuthal"

res_azim = ai.sigma_clip_ng(img_poisson, **k)
res_renorm = ai.sigma_clip_ng(img_poisson, normalization_factor=factor, **kwargs)
res_azim_renorm = ai.sigma_clip_ng(img_poisson, normalization_factor=factor, **k)
ref = ai.sigma_clip_ng(img_poisson, **kwargs)
npp = ai.sigma_clip_ng(img_poisson, **npp)
npa = ai.sigma_clip_ng(img_poisson, **npa)

ax = jupyter.plot1d(res_azim)
ax.plot(ref.radial, ref.sem, label="sem poisson unscale")
ax.plot(res_renorm.radial, res_renorm.sem*factor, "1",alpha=alpha, label="sem poisson, scaled")
ax.plot(res_azim.radial, res_azim.sem, alpha=alpha, label="sem azimuthal, unscaled")
ax.plot(res_azim_renorm.radial, res_azim_renorm.sem*factor, "1",alpha=alpha, label="sem azimuthal, scaled")

ax.plot(ref.radial, ref.std, label="std poisson unscale")
ax.plot(res_renorm.radial, res_renorm.std*factor, "1",alpha=alpha, label="std poisson, scaled")
ax.plot(res_azim.radial, res_azim.std, alpha=0.5, label="std azimuthal, unscaled")
ax.plot(res_azim_renorm.radial, res_azim_renorm.std*factor, "1",alpha=alpha, label="std azimuthal, scaled")

ax.legend()
_=ax.set_title("Sigma-clipping")
../../../_images/27f097703743af0566c0ae30a4c33f4b6c0f4a25e084f7ea1e1764ed8ace6a10.png
fig,ax = subplots()
what = "sum_variance"
ax.plot(npp.__getattribute__(what), label="poisson numpy")
ax.plot(ref.__getattribute__(what),"1",alpha=0.5, label="poisson opencl")
ax.plot(npa.__getattribute__(what), alpha=0.5, label="azimuth numpy")
ax.plot(res_azim.__getattribute__(what), "2", alpha=0.5, label="azimuth opencl")
ax.legend()
<matplotlib.legend.Legend at 0x7f2de0692c10>
../../../_images/268a77ca96377403d4c64f8b0e666f9f7f77d30218a21600b6b420c28fc30077.png

Merging two ensembles with the azimuthal error model#

With the azimuthal error model, the variance is not propagated from the pixels: it is measured from the dispersion of the pixel values around the mean of the bin. Each pixel \(i\) of value \(x_i\) contributes with a weight \(w_i\) (normalization × pixel-splitting coefficient) and the accumulated quantities for an ensemble \(A\) are:

\[\Omega_A = \sum_{i\in A} w_i \qquad \Omega^{(2)}_A = \sum_{i\in A} w_i^2 \qquad V_A = \sum_{i\in A} w_i x_i \qquad m_A = \frac{V_A}{\Omega_A} \qquad VV_A = \sum_{i\in A} w_i^2 (x_i - m_A)^2\]

The weighting in \(w^2\) of \(VV\) is the one which makes azimuthal and poisson error models equivalent, as shown above. The sem is then \(\sqrt{VV}/\Omega\) and the std \(\sqrt{VV/\Omega^{(2)}}\).

The integration is parallel: partial ensembles are accumulated independently and merged afterwards (within an OpenCL workgroup, the reduction is a binary tree). One needs \(VV_{A\cup B}\) from the accumulators of \(A\) and \(B\). Schubert & Gertz, Numerically stable parallel computation of (co-)variance, SSDBM 2018 (doi:10.1145/3221269.3223036) give such formulas for \(\sum w_i (x_i-m)^2\), i.e. with the same weights for the mean and the variance.

Where the cross-term comes from#

Let \(P = A\cup B\), and \(\delta_A = m_A - m_P\). Decomposing \(x_i - m_P = (x_i - m_A) + \delta_A\):

\[\sum_{i\in A} w_i^2 (x_i - m_P)^2 = VV_A + \Omega^{(2)}_A\,\delta_A^2 + 2\,\delta_A\,C_A \qquad\text{with}\qquad C_A = \sum_{i\in A} w_i^2 (x_i - m_A)\]
  • With Schubert & Gertz weighting, the equivalent of \(C_A\) is \(\sum w_i (x_i - m_A) = 0\) by definition of the mean: the cross-term vanishes, which is what simplifies their equations.

  • With the \(w^2\) weighting, \(C_A = \Omega^{(2)}_A\,(m^{(w^2)}_A - m_A)\) is the offset between the \(w^2\)-weighted and the \(w\)-weighted means. It vanishes only when both means coincide, e.g. when all weights within \(A\) are equal.

  • For a single element \(b\), \(C_b = 0\), but \(C_A \neq 0\): even adding one pixel to an ensemble depends on \(C_A\).

The cells below check this numerically on a small ensemble of Poisson-distributed values, sorted and split in two unequal parts to make the means of the two parts different.

rng = numpy.random.default_rng(seed=0)
size = 100
split = 90
x = numpy.sort(rng.poisson(100, size)).astype(numpy.float64)
w = rng.random(size) + 1

class Partition:
    "Reference statistics of an ensemble, calculated in two passes"
    def __init__(self, x, w=None):
        self.x = x
        self.w = w if w is not None else numpy.ones_like(x)
    @property
    def Omega(self):
        return self.w.sum()
    @property
    def Omega2(self):
        return (self.w*self.w).sum()
    @property
    def V(self):
        return (self.x*self.w).sum()
    @property
    def mean(self):
        return self.V/self.Omega
    @property
    def VV(self):
        return (self.w**2*(self.x - self.mean)**2).sum()
    @property
    def C(self):
        return (self.w**2*(self.x - self.mean)).sum()

A = Partition(x[:split], w[:split])
B = Partition(x[split:], w[split:])
P = Partition(x, w)
# This is the delta we are looking for:
D = P.VV - A.VV - B.VV
print(f"VV_P = {P.VV:.6f}, VV_A + VV_B = {A.VV + B.VV:.6f}, delta = {D:.6f}")
print(f"C_A = {A.C:.6f}, C_B = {B.C:.6f}")
VV_P = 27105.993555, VV_A + VV_B = 18693.387140, delta = 8412.606416
C_A = -49.699541, C_B = -3.192221

The formula currently used in pyFAI (_merge_azimuthal in OpenCL) is the one of Schubert & Gertz with \(\Omega^{(2)}_B\) in place of \(\Omega_B\), written with \(\Omega_A > \Omega_B\):

\[VV_{A\cup B} \approx VV_A + VV_B + \frac{\Omega^{(2)}_B\,(\Omega_A V_B - \Omega_B V_A)^2}{\Omega_P\,\Omega_A\,\Omega_B^2}\]

Expanding the decomposition above for both \(A\) and \(B\), with \(\Delta = m_A - m_B\), \(\delta_A = \Omega_B\,\Delta/\Omega_P\) and \(\delta_B = -\Omega_A\,\Delta/\Omega_P\), gives the exact expression:

\[VV_{A\cup B} = VV_A + VV_B + \Delta^2\,\frac{\Omega^{(2)}_A\,\Omega_B^2 + \Omega^{(2)}_B\,\Omega_A^2}{\Omega_P^2} + 2\,\Delta\,\frac{\Omega_B\,C_A - \Omega_A\,C_B}{\Omega_P}\]
Delta = A.mean - B.mean
current = B.Omega2*(A.Omega*B.V - B.Omega*A.V)**2/(B.Omega**2*A.Omega*P.Omega)
without_C = Delta**2*(A.Omega2*B.Omega**2 + B.Omega2*A.Omega**2)/P.Omega**2
exact = without_C + 2*Delta*(B.Omega*A.C - A.Omega*B.C)/P.Omega
print(f"delta (two-pass reference): {D:.6f}")
print(f"current pyFAI formula:      {current:.6f}  relative error {current/D-1:+.2e}")
print(f"exact formula without C:    {without_C:.6f}  relative error {without_C/D-1:+.2e}")
print(f"exact formula with C:       {exact:.6f}  relative error {exact/D-1:+.2e}")
delta (two-pass reference): 8412.606416
current pyFAI formula:      8304.555028  relative error -1.28e-02
exact formula without C:    8335.113356  relative error -9.21e-03
exact formula with C:       8412.606416  relative error +2.22e-16

No exact merge is possible with \((\Omega, \Omega^{(2)}, V, VV)\) only#

The cross-term is not a function of the four accumulators. Here are two ensembles \(A_1\) and \(A_2\) of three pixels with the same \(\Omega\), \(\Omega^{(2)}\), \(V\) and \(VV\) (solved numerically) but different \(C\): merged with the same pixel \(b\), they give different \(VV_{A\cup b}\). Whatever the formula, it cannot be exact for both.

from scipy.optimize import least_squares

A1 = Partition(numpy.array([90., 100., 115.]), numpy.array([1.0, 1.5, 1.2]))
def residual(p):
    A2 = Partition(p[:3], p[3:])
    return [A2.Omega - A1.Omega, A2.Omega2 - A1.Omega2, A2.V - A1.V, A2.VV - A1.VV]
sol = least_squares(residual, [85., 105., 112., 1.3, 1.1, 1.3], xtol=1e-15, ftol=1e-15, gtol=1e-15)
A2 = Partition(sol.x[:3], sol.x[3:])
xb, wb = 130., 1.0
for name, a in (("A1", A1), ("A2", A2)):
    print(f"{name}: x={a.x.round(4)}, w={a.w.round(4)}")
    print(f"    Omega={a.Omega:.6f} Omega2={a.Omega2:.6f} V={a.V:.6f} VV={a.VV:.6f} C={a.C:.6f}")
    print(f"    VV({name} ∪ b) = {Partition(numpy.r_[a.x, xb], numpy.r_[a.w, wb]).VV:.6f}")
A1: x=[ 90. 100. 115.], w=[1.  1.5 1.2]
    Omega=3.700000 Omega2=4.690000 V=378.000000 VV=395.763331 C=1.459459
    VV(A1 ∪ b) = 1023.268447
A2: x=[ 85.7284 104.872  111.0248], w=[0.9491 1.4276 1.3233]
    Omega=3.700000 Omega2=4.690000 V=378.000000 VV=395.763331 C=6.240616
    VV(A2 ∪ b) = 966.631395

Exact merge with a fifth accumulator#

Keeping \(C\) as an additional accumulator makes the merge exact. \(C\) itself is updated with:

\[C_{A\cup B} = C_A + C_B + \Omega^{(2)}_A\,\delta_A + \Omega^{(2)}_B\,\delta_B = C_A + C_B + \Delta\,\frac{\Omega^{(2)}_A\,\Omega_B - \Omega^{(2)}_B\,\Omega_A}{\Omega_P}\]

A single pixel is initialized with \(VV = C = 0\), so the same function performs the sequential accumulation (one pixel at a time) and the tree reduction. The formula is symmetric in \(A\) and \(B\): there is no need to order the ensembles by \(\Omega\).

One could store \(\sum w_i^2 x_i\) instead of \(C\), as the merge of plain sums is trivial, but \(C\) is the centered quantity: like \(VV\) compared to \(\sum w_i^2 x_i^2\), it avoids catastrophic cancellation.

The cell below compares both merge formulas with the two-pass reference on 1000 pixels, for a sequential accumulation and for a binary-tree reduction, then with an offset of \(10^8\) added to the signal to stress the numerical stability.

def single(x, w):
    "Accumulator (Omega, Omega2, V, VV, C) for a single pixel"
    return (w, w*w, w*x, 0.0, 0.0)

def merge_exact(a, b):
    Oa, O2a, Va, VVa, Ca = a
    Ob, O2b, Vb, VVb, Cb = b
    O = Oa + Ob
    Delta = Va/Oa - Vb/Ob
    VV = VVa + VVb + Delta*Delta*(O2a*Ob*Ob + O2b*Oa*Oa)/(O*O) + 2*Delta*(Ob*Ca - Oa*Cb)/O
    C = Ca + Cb + Delta*(O2a*Ob - O2b*Oa)/O
    return (O, O2a + O2b, Va + Vb, VV, C)

def merge_current(a, b):
    if a[0] < b[0]:
        a, b = b, a
    Oa, O2a, Va, VVa, _ = a
    Ob, O2b, Vb, VVb, _ = b
    O = Oa + Ob
    VV = VVa + VVb + O2b*(Oa*Vb - Ob*Va)**2/(O*Oa*Ob*Ob)
    return (O, O2a + O2b, Va + Vb, VV, numpy.nan)

def sequential(merge, x, w):
    acc = single(x[0], w[0])
    for xi, wi in zip(x[1:], w[1:]):
        acc = merge(acc, single(xi, wi))
    return acc[3]

def tree(merge, x, w):
    parts = [single(xi, wi) for xi, wi in zip(x, w)]
    while len(parts) > 1:
        parts = [merge(parts[i], parts[i+1]) if i + 1 < len(parts) else parts[i]
                 for i in range(0, len(parts), 2)]
    return parts[0][3]

x = rng.poisson(100, 1000).astype(numpy.float64)
w = rng.random(x.size) + 1
for offset in (0, 1e8):
    xo = x + offset
    ref = Partition(xo, w).VV
    print(f"Signal offset: {offset:g}, two-pass VV = {ref:.6f}")
    for name, merge in (("exact (with C)", merge_exact), ("current pyFAI", merge_current)):
        print(f"    {name:15s} sequential: {sequential(merge, xo, w)/ref-1:+.2e}   tree: {tree(merge, xo, w)/ref-1:+.2e}")
    # Same with plain sums: VV = Σw²x² - 2mΣw²x + m²Ω2
    m = (w*xo).sum()/w.sum()
    raw = (w*w*xo*xo).sum() - 2*m*(w*w*xo).sum() + m*m*(w*w).sum()
    print(f"    {'plain sums':15s} {raw/ref-1:+.2e}")
Signal offset: 0, two-pass VV = 233642.052367
    exact (with C)  sequential: +6.66e-16   tree: +2.22e-16
    current pyFAI   sequential: -3.02e-04   tree: -9.68e-02
    plain sums      -3.04e-14
Signal offset: 1e+08, two-pass VV = 233642.052366
    exact (with C)  sequential: +2.33e-10   tree: -2.54e-11
    current pyFAI   sequential: -3.02e-04   tree: -9.68e-02
    plain sums      -1.83e-02

Conclusion#

  • With the \(w^2\) weighting of the variance, the merge of two ensembles involves the cross-term \(C = \sum w_i^2 (x_i - m)\), which does not vanish, contrary to the case treated by Schubert & Gertz. This holds also when adding a single pixel to an ensemble.

  • No formula based only on \((\Omega, \Omega^{(2)}, V, VV)\) can be exact.

  • With \(C\) as a fifth accumulator, the merge is exact, symmetric and numerically stable.

  • The current formula is a good approximation for a sequential accumulation, but its error grows when merging ensembles of comparable sizes, as in a tree reduction.