Differential Phase Processing#

import matplotlib.pyplot as plt
import xarray as xr
import radarpolarimetry as rp

Differential Phase Overview#

Load Dataset (Overview)#

Showing different problems with differential phase. We use 4.5° scan to show bright band backscattering.

stage = "02_system_phidp"
dtime = "20170810T1100"
fname = rp.data_path() / "lecture" / f"{stage}_{dtime}.zarr"
swp = xr.open_dataset(fname)

Inspect Dataset#

display(swp.load())
<xarray.Dataset> Size: 55MB
Dimensions:            (azimuth: 360, range: 750)
Coordinates: (12/16)
  * azimuth            (azimuth) float64 3kB 0.5 1.5 2.5 ... 357.5 358.5 359.5
    elevation          (azimuth) float64 3kB 0.9998 0.9998 ... 0.9998 0.9998
    time               (azimuth) datetime64[ns] 3kB 2017-08-10T11:03:42.69900...
  * range              (range) float32 3kB 100.0 300.0 ... 1.497e+05 1.499e+05
    bins               (azimuth, range) float32 1MB 100.0 300.0 ... 1.499e+05
    gr                 (azimuth, range) float64 2MB 99.98 300.0 ... 1.498e+05
    ...                 ...
    altitude           float64 8B 99.5
    crs_wkt            int64 8B 0
    latitude           float64 8B 50.73
    longitude          float64 8B 7.072
    NOISE_LEVEL        float64 8B -24.7
    sweep_mode         <U20 80B 'azimuth_surveillance'
Data variables: (12/30)
    CMAP               (azimuth, range) float64 2MB 12.0 8.0 nan ... 771.0 771.0
    CMAP2              (azimuth, range) uint8 270kB 1 1 0 1 0 1 ... 0 0 0 0 0 0
    DBTH               (azimuth, range) float64 2MB 40.28 24.72 ... 6.652 4.644
    DBTV               (azimuth, range) float64 2MB 41.29 30.24 ... 5.648 nan
    DBZH               (azimuth, range) float64 2MB nan 17.69 18.7 ... nan nan
    DBZV               (azimuth, range) float64 2MB nan 19.7 20.2 ... nan nan
    ...                 ...
    SYSPHI4            (azimuth) float64 3kB 85.85 83.65 85.35 ... 86.25 84.55
    VRADH              (azimuth, range) float64 2MB nan 0.1771 ... nan nan
    VRADV              (azimuth, range) float64 2MB nan 0.1771 ... nan nan
    WRADH              (azimuth, range) float64 2MB nan 0.7084 ... nan nan
    WRADV              (azimuth, range) float64 2MB nan 0.4206 ... nan nan
    ZDR                (azimuth, range) float64 2MB nan 1.45 -0.7 ... nan nan
Attributes:
    source:         gamic
    ant_gain_h:     44.0
    ant_gain_v:     44.0
    noise_power_h:  -1.39495
    noise_power_v:  -1.09706
    rx_loss_h:      0.73
    rx_loss_v:      0.73
bit = 0
no_clutter_bb = ((swp.CMAP.astype("uint16") >> bit) & 0x1 == 0) & (swp.CMAP2 == 0)
swpm = swp.where(no_clutter_bb & (swp.RHOHV >= 0.95) & (swp.DBTH > 0))
phimask = swpm.PHIDP_OC1
fig = plt.figure()
ax = fig.add_subplot()
rp.plot_moment(phimask, ax=ax)
<matplotlib.collections.QuadMesh at 0x7f3a9c387620>
../../_images/71cd9dcff49413fb7c04500dcbb2513c31219dee7411b6892cce8f474e25ea3e.png

Calculate Median#

medwin = 21
phimed = phimask.pipe(rp.filter_data, medwin)
phimed.attrs = phimask.attrs
fig = plt.figure()
ax = fig.add_subplot()
rp.plot_moment(phimed, ax=ax)
<matplotlib.collections.QuadMesh at 0x7f3a9c4df9d0>
../../_images/98bd01062b0c903ebb29394bfc09bc3364fec21f661938b3c2490a7f0683e3f2.png

Plot moments#

swpx = swpm.sel(range=slice(0, 100e3))
rp.plot_2by2(swpx, moments=["DBTH", "PHIDP_OC1", "ZDR", "RHOHV"])
../../_images/e3a81ad38a332d7a99e89a81575cf2b45493400204c0c4b731a2387a79480704.png

Visualize single radial#

azi = 200

swpx = swpm.sel(range=slice(0, 80e3))
phimedx = phimed.sel(range=slice(0, 80e3))
fig = plt.figure(figsize=(10, 8))

ax = fig.add_subplot(232)
rp.plot_moment(swpx.DBZH, ax=ax)
az = swpx.isel(azimuth=azi)
plt.gca().plot(az.x, az.y, "k", lw=0.85)


# create host axis with ylabel
ax0 = rp.create_lineplot(fig, subplot=212, ylabel=r"$PHI_{DP}$ (deg)")
ax0.set_ylim(-25, 50)

# plot line on first axis
line = phimedx[azi].plot(c="b", marker=".", label=r"smoothed $PHI_{DP}$ (deg)", ax=ax0)
line = swp.PHIDP_OC1[azi].plot(c="b", lw=0.5, label=r"raw $PHI_{DP}$ (deg)", ax=ax0)

# add twin axis of host axis with label and position
ax1 = rp.add_axis(ax0, ylabel=r"$RHO_{HV}$", pos=1.0)
ax1.grid()
# add another twin axis of host axis with label and position
ax2 = rp.add_axis(ax0, ylabel=r"$UZ_H$ (dBZ)", pos=1.1)
# plot line on second axis
# line = swpx.RHOHV_NC[azi].plot(c="r", label=r"$RHO_{HV}$ - nc", ax=ax1)
line = swpx.RHOHV[azi].plot(c="r", label=r"$RHO_{HV}$", lw=1.0, ax=ax1)
line = swp.RHOHV[azi].plot(c="r", label=r"$RHO_{HV}$", lw=0.25, ax=ax1)
# plot line on third axis
line = swpx.DBZH[azi].plot(c="g", label=r"$UZ_H$ (dBZ)", ax=ax2)
line = swp.DBZH[azi].plot(c="g", label=r"$UZ_H$ (dBZ)", lw=0.25, ax=ax2)

# set title
ax0.set_title(f"{swp.time.min().values} Azi {azi}")
ax1.set_title("")
ax2.set_title("")

# legend
h1, l1 = ax0.get_legend_handles_labels()
h2, l2 = ax1.get_legend_handles_labels()
h3, l3 = ax2.get_legend_handles_labels()
ax1.legend(h1 + h2 + h3, l1 + l2 + l3, loc="lower right")
<matplotlib.legend.Legend at 0x7f3a9d9f8ad0>
../../_images/8dc1f3b3b75bb33b6b4f9ee5f66c5a26779898bb4361c3d39575b1541d6c1499.png

Exercise 1: Use other azimuths and compare#

Go back to cell above and check other azimuths and compare. Also play with different masking thresholds.

Phase Processing Workflow#

Here we now start the complete workflow of masking, noise filtering and smoothing. Finally we save the corrected differential phase.

Load dataset (workflow)#

swp = xr.open_dataset(fname)

Mask data using DBZH, RHOHV, CMAP/CMAP2#

Depending on the availability of CMAP/CMAP2 moments the data is masked to remove invalid data points.

CMAP = swp.CMAP.astype("uint16")
noise = ((CMAP >> 0) & 0x1).astype(bool)
clutter = ((CMAP >> 2) & 0x1).astype(bool)
masked = noise | clutter
valid = ~masked
CMAP2 = swp.CMAP2
clutter2 = CMAP2 == 1
bb = CMAP == 2
masked2 = clutter2 | bb
valid2 = ~masked2
val = valid & valid2
swp_masked = swp.where(val).where(swp.RHOHV >= 0.95).where(swp.DBZH >= 0)
azi = 100
fig = plt.figure(figsize=(10, 8))
ax = fig.add_subplot()
rp.plot_moment(swp_masked.DBTH, ax=ax)
az = swp_masked.isel(azimuth=azi)
display(az)
plt.gca().plot(az.x, az.y, "k")
<xarray.Dataset> Size: 213kB
Dimensions:            (range: 750)
Coordinates: (12/16)
  * range              (range) float32 3kB 100.0 300.0 ... 1.497e+05 1.499e+05
    bins               (range) float32 3kB 100.0 300.0 ... 1.497e+05 1.499e+05
    gr                 (range) float64 6kB 99.98 300.0 ... 1.496e+05 1.498e+05
    rays               (range) float64 6kB 100.5 100.5 100.5 ... 100.5 100.5
    x                  (range) float64 6kB 98.31 294.9 ... 1.471e+05 1.473e+05
    y                  (range) float64 6kB -18.22 -54.66 ... -2.73e+04
    ...                 ...
    elevation          float64 8B 0.9998
    latitude           float64 8B 50.73
    longitude          float64 8B 7.072
    NOISE_LEVEL        float64 8B -24.7
    sweep_mode         <U20 80B 'azimuth_surveillance'
    time               datetime64[ns] 8B 2017-08-10T11:03:52.751000
Data variables: (12/30)
    CMAP               (range) float64 6kB nan nan nan nan ... nan nan nan nan
    CMAP2              (range) float32 3kB nan nan nan nan ... nan nan nan nan
    DBTH               (range) float64 6kB nan nan nan nan ... nan nan nan nan
    DBTV               (range) float64 6kB nan nan nan nan ... nan nan nan nan
    DBZH               (range) float64 6kB nan nan nan nan ... nan nan nan nan
    DBZV               (range) float64 6kB nan nan nan nan ... nan nan nan nan
    ...                 ...
    SYSPHI4            (range) float64 6kB nan nan nan nan ... nan nan nan nan
    VRADH              (range) float64 6kB nan nan nan nan ... nan nan nan nan
    VRADV              (range) float64 6kB nan nan nan nan ... nan nan nan nan
    WRADH              (range) float64 6kB nan nan nan nan ... nan nan nan nan
    WRADV              (range) float64 6kB nan nan nan nan ... nan nan nan nan
    ZDR                (range) float64 6kB nan nan nan nan ... nan nan nan nan
Attributes:
    source:         gamic
    ant_gain_h:     44.0
    ant_gain_v:     44.0
    noise_power_h:  -1.39495
    noise_power_v:  -1.09706
    rx_loss_h:      0.73
    rx_loss_v:      0.73
[<matplotlib.lines.Line2D at 0x7f3a22508f50>]
../../_images/820848e03f02231a01c1b80607c7ab125948a4c12d4d2d1cb635d3ca26b2c690.png

“Blend” RHOHV#

swp_masked = swp_masked.assign(
    RHOHV_SMOOTH=swp_masked.RHOHV.rolling(range=11, min_periods=2, center=True).median(
        skipna=True
    )
)
swp_masked.RHOHV_SMOOTH.wrl.vis.plot()
<matplotlib.collections.QuadMesh at 0x7f3a21ab96d0>
../../_images/a65a70199754fbf090988bd44e6d23bbfe947402601ff8e1443b2b48b4949902.png
fig = plt.figure(figsize=(10, 8))
# create host axis with ylabel
ax0 = rp.create_lineplot(fig, subplot=111, ylabel=r"$PHI_{DP}$ (deg)")

# plot line on first axis
line = swp_masked.PHIDP_OC1[azi].plot(
    c="b", marker=".", lw=0.5, label=r"raw $PHI_{DP}$ (deg)", ax=ax0
)

ax0.set_ylim(-5, 40)
# add twin axis of host axis with label and position
ax1 = rp.add_axis(ax0, ylabel=r"$RHO_{HV}$", pos=1.0)
ax1.set_ylim(0, 1.1)
# add another twin axis of host axis with label and position
ax2 = rp.add_axis(ax0, ylabel=r"$UZ_H$ (dBZ)", pos=1.1)
ax2.set_ylim(0, 50)
# plot line on second axis
line = swp_masked.RHOHV_SMOOTH[azi].plot(
    c="r", marker=".", label=r"smoothed $RHO_{HV}$", lw=0.75, ax=ax1
)
# line = swp.RHOHV_NC[azi].plot(c="r", label=r"$RHO_{HV}$ - nc", ax=ax1)
# plot line on third axis
line = swp_masked.DBZH[azi].plot(c="g", marker=".", label=r"$UZ_H$ (dBZ)", ax=ax2)

# set title
ax0.set_title(f"{swp.time.min().values} Azi {azi}")
ax1.set_title("")
ax2.set_title("")

# legend
h1, l1 = ax0.get_legend_handles_labels()
h2, l2 = ax1.get_legend_handles_labels()
h3, l3 = ax2.get_legend_handles_labels()
ax1.legend(h1 + h2 + h3, l1 + l2 + l3)
<matplotlib.legend.Legend at 0x7f3a22611d10>
../../_images/9a58e08bebb3a22761e5ec9251a61bb8135c897bf7f5644c26a742108ebf8a50.png

Filtering and Smoothing#

The phase data is filtered using a median filter and smoothed with a gaussian kernel.

We use the standard method of Phase Offset Correction (first consecutive N precipitationg bins)

Median Filtering#

medwin = 41
# median filtering 2d
phimed = swp_masked.PHIDP_OC1.copy()
phimed = phimed.pipe(rp.filter_data, medwin)
phimed.attrs = swp_masked.PHIDP_OC1.attrs

Smoothing#

# gaussian convolution 1d - smoothing
# play with sigma and kwidth
kwidth = 41
sigma = 10
gkern = rp.gauss_kernel(kwidth, sigma)

phiclean = phimed.pipe(rp.smooth_data, gkern)
phiclean.attrs = phimed.attrs
display(phiclean)
<xarray.DataArray 'PHIDP_OC1' (azimuth: 360, range: 750)> Size: 2MB
array([[      nan,       nan,       nan, ...,       nan,       nan,
              nan],
       [      nan,       nan, 0.5713065, ...,       nan,       nan,
              nan],
       [      nan,       nan,       nan, ...,       nan,       nan,
              nan],
       ...,
       [      nan,       nan,       nan, ...,       nan,       nan,
              nan],
       [      nan,       nan,       nan, ...,       nan,       nan,
              nan],
       [      nan,       nan,       nan, ...,       nan,       nan,
              nan]], shape=(360, 750))
Coordinates: (12/16)
  * azimuth      (azimuth) float64 3kB 0.5 1.5 2.5 3.5 ... 357.5 358.5 359.5
    elevation    (azimuth) float64 3kB 0.9998 0.9998 0.9998 ... 0.9998 0.9998
    time         (azimuth) datetime64[ns] 3kB 2017-08-10T11:03:42.699000 ... ...
  * range        (range) float32 3kB 100.0 300.0 500.0 ... 1.497e+05 1.499e+05
    bins         (azimuth, range) float32 1MB 100.0 300.0 ... 1.499e+05
    gr           (azimuth, range) float64 2MB 99.98 300.0 ... 1.498e+05
    ...           ...
    altitude     float64 8B 99.5
    crs_wkt      int64 8B 0
    latitude     float64 8B 50.73
    longitude    float64 8B 7.072
    NOISE_LEVEL  float64 8B -24.7
    sweep_mode   <U20 80B 'azimuth_surveillance'
Attributes:
    format:         UV16
    is_dft:         0
    unit:           deg
    units:          degrees
    long_name:      Differential phase HV
    standard_name:  radar_differential_phase_hv
    _Undetect:      0.0
time = swp.time.min().values
# create figure
fig = plt.figure(figsize=(10, 8))
# create host axis with ylabel
ax0 = rp.create_lineplot(fig, subplot=111, ylabel=r"$PHI_{DP}$ (deg)")
# set title
ax0.set_title(f"{time} Azi {azi}")
# plot line on first axis
# line = swp.PHIDP_OC1[azi].plot(c="lightblue", label=r"raw $PHI_{DP}$ (deg)", ax=ax0)
line = swp_masked.PHIDP_OC1[azi].plot(
    c="b", marker="*", label=r"masked $PHI_{DP}$ (deg)", ax=ax0
)
line = phimed[azi].plot(
    c="y", marker=".", label=r"smoothed $PHI_{DP}$ (deg)", zorder=10, ax=ax0
)

# add twin axis of host axis with label and position
ax1 = rp.add_axis(ax0, ylabel=r"$RHO_{HV}$", pos=1.0)
# add another twin axis of host axis with label and position
ax2 = rp.add_axis(ax0, ylabel=r"$Z_H$ (dBZ)", pos=1.1)
# plot line on second axis
line = swp_masked.RHOHV_SMOOTH[azi].plot(
    c="r", marker=".", label=r"smoothed $RHO_{HV}$", zorder=0, ax=ax1
)
# line = swp_masked.RHOHV[azi].plot(c="darkred", marker=".", label=r"smoothed $RHO_{HV}$", zorder=0, ax=ax1)
# plot line on third axis
line = swp_masked.DBTH[azi].plot(
    c="g", marker=".", label=r"$Z_H$ (dBZ)", zorder=0, ax=ax2
)
ax0.set_ylim(-5, 40)
line = phiclean[azi].plot(
    c="k", marker=".", label=r"clean$PHI_{DP}$ (deg)", zorder=10, ax=ax0
)
h1, l1 = ax0.get_legend_handles_labels()
h2, l2 = ax1.get_legend_handles_labels()
h3, l3 = ax2.get_legend_handles_labels()

ax1.legend(h1 + h2 + h3, l1 + l2 + l3)

plt.show()
../../_images/0eb6817e6de136dbe4dc83e9492eb4d477f3c37c7642e0b932598b4922a56b14.png

Exercise 2: Play with medwin, kwidth and sigma#

Go back above and play with different values for smoothing.

2d Filtering using xr_rolling#

Can clean out outliers of phase in single rays.

%%time
window = 41
window2 = 3
phi_median = swp_masked.PHIDP_OC1.copy().pipe(
    rp.xr_rolling, window, window2=window2, method="median", skipna=True, min_periods=3
)
# gaussian convolution 1d - smoothing
# play with sigma and kwidth
kwidth = 41
sigma = 10
gkern = rp.gauss_kernel(kwidth, sigma)

phi_clean = phi_median.pipe(rp.smooth_data, gkern)
phi_clean.attrs = phimed.attrs
phi_clean.attrs = swp_masked.PHIDP_OC1.attrs
CPU times: user 1.51 s, sys: 402 ms, total: 1.92 s
Wall time: 1.92 s
time = swp.time.min().values
# create figure
fig = plt.figure(figsize=(10, 8))
# create host axis with ylabel
ax0 = rp.create_lineplot(fig, subplot=111, ylabel=r"$PHI_{DP}$ (deg)")
# set title
ax0.set_title(f"{time} Azi {azi}")
# plot line on first axis
line = swp.PHIDP_OC1[azi].plot(c="lightblue", label=r"raw $PHI_{DP}$ (deg)", ax=ax0)
line = swp_masked.PHIDP_OC1[azi].plot(
    c="b", marker="*", label=r"masked $PHI_{DP}$ (deg)", ax=ax0
)


# add twin axis of host axis with label and position
ax1 = rp.add_axis(ax0, ylabel=r"$RHO_{HV}$", pos=1.0)
# add another twin axis of host axis with label and position
ax2 = rp.add_axis(ax0, ylabel=r"$Z_H$ (dBZ)", pos=1.1)
# plot line on second axis
line = swp_masked.RHOHV_SMOOTH[azi].plot(c="r", label=r"$RHO_{HV}$", zorder=0, ax=ax1)
# plot line on third axis
line = swp_masked.DBTH[azi].plot(c="g", label=r"$Z_H$ (dBZ)", zorder=0, ax=ax2)
ax0.set_ylim(-5, 40)
line = phiclean[azi].plot(
    c="k", marker=".", label=r"clean $PHI_{DP}$ (deg)", zorder=10, ax=ax0
)
line = phi_clean[azi].plot(
    c="y", marker=".", label=r"clean 2D $PHI_{DP}$ (deg)", zorder=10, ax=ax0
)
h1, l1 = ax0.get_legend_handles_labels()
h2, l2 = ax1.get_legend_handles_labels()
h3, l3 = ax2.get_legend_handles_labels()

ax1.legend(h1 + h2 + h3, l1 + l2 + l3)

plt.show()
../../_images/e96948d6ca87bc188e512d40dfd378b5238444b1df9450196afe1f4ab63b18fc.png

Exercise 3: Play with window sizes#

Go back above and play with different window sizes and smoothing parameters.

Overview Plot#

fig, axes = plt.subplots(2, 2, sharex=True, sharey=True, figsize=(12, 10))
axes = axes.flat
phimed.wrl.vis.plot(vmin=-5, vmax=40, ax=axes[0])
phi_median.wrl.vis.plot(vmin=-5, vmax=40, ax=axes[1])
phiclean.wrl.vis.plot(vmin=-5, vmax=40, ax=axes[2])
phi_clean.wrl.vis.plot(vmin=-5, vmax=40, ax=axes[3])
plt.tight_layout()
../../_images/af8a0e32e23a02022cb73513b7769621bc2d8f95a4ea7e9fa9cd0774def92f7a.png

Add and save data#

swp = swp.assign({"PHIDP_CORR1": phimed})
swp = swp.assign({"PHIDP_CORR2": phi_median})
swp = swp.assign({"PHIDP_CORR3": phiclean})
swp = swp.assign({"PHIDP_CORR4": phi_clean})
dim0 = "azimuth"
dtime = swp.time.min(dim0).dt.floor("5min").dt.strftime("%Y%m%dT%H%M").values
stage = "03_phidp"
fname = rp.data_path() / "lecture" / f"{stage}_{dtime}.zarr"
print(fname)
swp.to_zarr(fname, mode="w", zarr_version=2)
/builds/_NRRzFk9l/0/grp_meteo/radar/radarpolarimetry/data/lecture/03_phidp_20170810T1100.zarr
<xarray.backends.zarr.ZarrStore at 0x7f3a2284ce00>