{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cda28b8e",
   "metadata": {},
   "source": [
    "# Supervised Wishart evaluation on real data\n",
    "\n",
    "This notebook builds C-PolSARPro training inputs from the real San Francisco label map,\n",
    "runs the legacy supervised Wishart classifier on the same ROI as the Python version,\n",
    "and compares the two outputs.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b746136",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "import subprocess\n",
    "from pathlib import Path\n",
    "\n",
    "import dask.array as da\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import xarray as xr\n",
    "from dask.diagnostics import ProgressBar\n",
    "from scipy.ndimage import label as cc_label\n",
    "\n",
    "from polsarpro.classification import (\n",
    "    _label_training_clusters,\n",
    "    _update_wishart_class_centers,\n",
    "    wishart_supervised,\n",
    ")\n",
    "from polsarpro.dev.io import read_psp_bin\n",
    "from polsarpro.io import open_netcdf_beam\n",
    "from polsarpro.util import S_to_T3, boxcar\n",
    "\n",
    "c_psp_root = Path(\"/home/c_psp/Soft\")\n",
    "c_classifier = c_psp_root / \"bin\" / \"data_process_sngl\" / \"wishart_supervised_classifier.exe\"\n",
    "\n",
    "input_alos_data = Path(\"/data/psp/test_files/SAN_FRANCISCO_ALOS1_slc.nc\")\n",
    "input_test_dir = Path(\"/data/psp/SAN_FRANCISCO_ALOS1/T3\")\n",
    "label_file = Path(\"/polsarpro-dev/data/sample_labels_sf_alos1.nc\")\n",
    "mask_file = input_test_dir / \"mask_valid_pixels.bin\"\n",
    "\n",
    "output_test_dir = Path(\"/data/psp/res/wishart_supervised_real_data\")\n",
    "output_test_dir.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "boxcar_size = [7, 7]\n",
    "\n",
    "\n",
    "def write_config_file(directory: Path, nrow: int, ncol: int) -> Path:\n",
    "    \"\"\"Create the minimal PSP config.txt needed by read_psp_bin.\"\"\"\n",
    "    directory = Path(directory)\n",
    "    directory.mkdir(parents=True, exist_ok=True)\n",
    "    config_path = directory / \"config.txt\"\n",
    "    config_path.write_text(\n",
    "        f\"\"\"Nrow\n",
    "{nrow}\n",
    "---------\n",
    "Ncol\n",
    "{ncol}\n",
    "---------\n",
    "PolarCase\n",
    "monostatic\n",
    "---------\n",
    "PolarType\n",
    "full\n",
    "\"\"\"\n",
    "    )\n",
    "    return config_path\n",
    "\n",
    "\n",
    "def write_area_file(training_labels: np.ndarray, file_name: Path) -> dict[int, int]:\n",
    "    \"\"\"Write the text file consumed by create_class_map() in the C implementation.\"\"\"\n",
    "    file_name = Path(file_name)\n",
    "    file_name.parent.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "    # The legacy C parser ignores the explicit class value written in the file and\n",
    "    # instead assigns semantic labels from the class-block position (1, 2, 3, ...).\n",
    "    # To preserve original labels when a crop removes an intermediate class, we must\n",
    "    # still emit empty blocks for missing class ids.\n",
    "    class_values = list(range(1, int(training_labels.max()) + 1))\n",
    "    lines = [\"classes\", str(len(class_values))]\n",
    "    polygons_per_class = {}\n",
    "\n",
    "    for class_value in class_values:\n",
    "        regions, nregions = cc_label(training_labels == class_value)\n",
    "        polygons_per_class[class_value] = int(nregions)\n",
    "        lines.extend([\"class\", str(class_value), \"areas\", str(nregions)])\n",
    "\n",
    "        for region_id in range(1, nregions + 1):\n",
    "            coords = np.argwhere(regions == region_id)\n",
    "            lines.extend([\"area\", str(region_id), str(len(coords))])\n",
    "            for point_id, (yy, xx) in enumerate(coords, start=1):\n",
    "                lines.extend([\"pt\", str(point_id), str(float(yy)), \"x\", str(float(xx))])\n",
    "\n",
    "    file_name.write_text(\"\\n\".join(lines) + \"\\n\")\n",
    "    return polygons_per_class\n",
    "\n",
    "\n",
    "def write_cluster_file(filtered_t3: xr.Dataset, training_labels: np.ndarray, file_name: Path):\n",
    "    \"\"\"Write the C cluster binary: ncluster as float32, then one T3 center per cluster.\"\"\"\n",
    "    file_name = Path(file_name)\n",
    "    file_name.parent.mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "    lab, cluster_classes = _label_training_clusters(training_labels.astype(np.int32, copy=False))\n",
    "    lab_da = da.from_array(lab, chunks=lab.shape)\n",
    "    centers = _update_wishart_class_centers(filtered_t3, lab_da, nclass=len(cluster_classes) - 1).compute()\n",
    "\n",
    "    rows = []\n",
    "    for center in np.asarray(centers):\n",
    "        rows.append(\n",
    "            [\n",
    "                np.real(center[0, 0]),\n",
    "                np.real(center[0, 1]),\n",
    "                np.imag(center[0, 1]),\n",
    "                np.real(center[0, 2]),\n",
    "                np.imag(center[0, 2]),\n",
    "                np.real(center[1, 1]),\n",
    "                np.real(center[1, 2]),\n",
    "                np.imag(center[1, 2]),\n",
    "                np.real(center[2, 2]),\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    with open(file_name, \"wb\") as f:\n",
    "        np.array([len(rows)], dtype=np.float32).tofile(f)\n",
    "        np.asarray(rows, dtype=np.float32).ravel().tofile(f)\n",
    "\n",
    "    return lab, cluster_classes, np.asarray(rows, dtype=np.float32)\n",
    "\n",
    "\n",
    "def summarize_agreement(reference: np.ndarray, candidate: np.ndarray, labels: np.ndarray):\n",
    "    summary = {\n",
    "        \"overall_agreement\": float((reference == candidate).mean()),\n",
    "        \"agreement_outside_training\": float((reference[labels == 0] == candidate[labels == 0]).mean()),\n",
    "        \"agreement_on_training\": float((reference[labels > 0] == candidate[labels > 0]).mean()),\n",
    "    }\n",
    "\n",
    "    classes = np.unique(np.concatenate([reference.ravel(), candidate.ravel()]))\n",
    "    per_class = []\n",
    "    for class_value in classes:\n",
    "        mask = reference == class_value\n",
    "        if mask.any():\n",
    "            per_class.append((int(class_value), int(mask.sum()), float((candidate[mask] == class_value).mean())))\n",
    "\n",
    "    return summary, per_class\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "475a1b89",
   "metadata": {},
   "source": [
    "## Prepare the image and the C training files\n",
    "\n",
    "This notebook uses the full image by default. If we later need a faster turnaround,\n",
    "we can uncomment the optional manual crop below and apply the same slice to the image\n",
    "and the label map.\n",
    "\n",
    "If a manual crop is used, small differences may appear near the crop borders. The\n",
    "legacy C executable effectively averages with surrounding context from the original\n",
    "scene, while the Python workflow here applies the boxcar on the already cropped data.\n",
    "This is a user-side comparison detail rather than a critical algorithm issue, and we\n",
    "do not change the internal Python workflow for it because that would complicate and\n",
    "break the current usage pattern.\n",
    "\n",
    "For the legacy comparison we intentionally point the C executable to the PolSARPro `T3`\n",
    "directory and run it with `-iodf T3`. This avoids the ambiguous `S2` path, where the\n",
    "legacy code switches between `T3` and `T4` depending on `PolarCase` in `config.txt`.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d9a17b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_ds = xr.open_dataset(label_file)\n",
    "labels_full = labels_ds[\"labels\"].astype(np.int16)\n",
    "\n",
    "row1, row2 = 0, labels_full.sizes[\"y\"]\n",
    "col1, col2 = 0, labels_full.sizes[\"x\"]\n",
    "\n",
    "# Optional manual crop for quicker iteration if needed later.\n",
    "# row1, row2 = 8000, 12000\n",
    "# col1, col2 = 200, 1100\n",
    "\n",
    "labels_roi = labels_full.isel(y=slice(row1, row2), x=slice(col1, col2)).copy()\n",
    "labels_np = labels_roi.values.astype(np.int16, copy=False)\n",
    "\n",
    "print(f\"ROI rows: {row1}:{row2} ({row2-row1})\")\n",
    "print(f\"ROI cols: {col1}:{col2} ({col2-col1})\")\n",
    "print(\"Semantic classes:\", np.unique(labels_np[labels_np > 0]).tolist())\n",
    "print(\"Training pixels:\", int((labels_np > 0).sum()))\n",
    "\n",
    "S_roi = open_netcdf_beam(input_alos_data).isel(y=slice(row1, row2), x=slice(col1, col2))\n",
    "T3_roi = boxcar(S_to_T3(S_roi), dim_az=boxcar_size[0], dim_rg=boxcar_size[1])\n",
    "\n",
    "area_file = output_test_dir / \"wishart_training_areas.txt\"\n",
    "cluster_file = output_test_dir / \"wishart_training_clusters.bin\"\n",
    "\n",
    "polygons_per_class = write_area_file(labels_np, area_file)\n",
    "lab, cluster_classes, cluster_centers = write_cluster_file(T3_roi, labels_np, cluster_file)\n",
    "\n",
    "print(\"Polygons per class:\", polygons_per_class)\n",
    "print(\"Connected components:\", len(cluster_classes) - 1)\n",
    "print(\"Area file:\", area_file)\n",
    "print(\"Cluster file:\", cluster_file)\n",
    "print(\"First cluster center:\")\n",
    "cluster_centers[0]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "35a44c4d",
   "metadata": {},
   "source": [
    "## Run the C implementation\n",
    "\n",
    "The `area` file only restores cluster-to-class mapping in the legacy code; the actual\n",
    "cluster statistics come from the binary file we just generated from the Python-side T3 means.\n",
    "\n",
    "The legacy executable also returns exit code `1` even when it completes successfully,\n",
    "so the cell below accepts both `0` and `1` as non-error return codes.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a78b5316",
   "metadata": {},
   "outputs": [],
   "source": [
    "c_cmd = [\n",
    "    str(c_classifier),\n",
    "    \"-id\", str(input_test_dir),\n",
    "    \"-od\", str(output_test_dir),\n",
    "    \"-iodf\", \"T3\",\n",
    "    \"-af\", str(area_file),\n",
    "    \"-nwr\", str(boxcar_size[0]),\n",
    "    \"-nwc\", str(boxcar_size[1]),\n",
    "    \"-ofr\", str(row1),\n",
    "    \"-ofc\", str(col1),\n",
    "    \"-fnr\", str(row2 - row1),\n",
    "    \"-fnc\", str(col2 - col1),\n",
    "    \"-cf\", str(cluster_file),\n",
    "    \"-bmp\", \"0\",\n",
    "    \"-errf\", \"/tmp/MemoryAllocError.txt\",\n",
    "    \"-mask\", str(mask_file),\n",
    "]\n",
    "\n",
    "print(\" \".join(c_cmd))\n",
    "completed = subprocess.run(c_cmd, check=False)\n",
    "if completed.returncode not in (0, 1):\n",
    "    raise RuntimeError(f\"C classifier failed with exit code {completed.returncode}\")\n",
    "write_config_file(output_test_dir, nrow=row2 - row1, ncol=col2 - col1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2148dae5",
   "metadata": {},
   "source": [
    "## Run the Python implementation on the same ROI\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "901bc686",
   "metadata": {},
   "outputs": [],
   "source": [
    "out_file = output_test_dir / \"wishart_supervised.nc\"\n",
    "\n",
    "# remove if file already exists as xarray will not overwrite existing files and will error instead\n",
    "if out_file.exists():\n",
    "    out_file.unlink()\n",
    "\n",
    "with ProgressBar():\n",
    "    out_py = wishart_supervised(\n",
    "        S_roi,\n",
    "        training_labels=labels_roi,\n",
    "        boxcar_size=boxcar_size,\n",
    "    ).to_netcdf(out_file)\n",
    "\n",
    "out_py = xr.open_dataset(out_file) \n",
    "class_py = out_py[\"wishart_supervised_class\"].values.astype(np.int16)\n",
    "class_py\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6973d522",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(out_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ab0a7c8",
   "metadata": {},
   "source": [
    "## Numerical comparison\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1823326e",
   "metadata": {},
   "outputs": [],
   "source": [
    "c_output_path = output_test_dir / f\"wishart_supervised_class_{boxcar_size[0]}x{boxcar_size[1]}.bin\"\n",
    "class_c = read_psp_bin(c_output_path).astype(np.int16)\n",
    "summary, per_class = summarize_agreement(class_c, class_py, labels_np)\n",
    "\n",
    "print(\"Agreement summary\")\n",
    "for key, value in summary.items():\n",
    "    print(f\"- {key}: {value:.4f}\")\n",
    "\n",
    "print(\"\\nAgreement by C class\")\n",
    "for class_value, npix, agreement in per_class:\n",
    "    print(f\"- class {class_value}: {npix} px, agreement={agreement:.4f}\")\n",
    "\n",
    "print(\"\\nClass histograms\")\n",
    "for source_name, arr in [(\"C\", class_c), (\"Python\", class_py)]:\n",
    "    values, counts = np.unique(arr, return_counts=True)\n",
    "    print(source_name, dict(zip(values.tolist(), counts.tolist())))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc5eba77",
   "metadata": {},
   "source": [
    "## Quick-look plots\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9e6e489",
   "metadata": {},
   "outputs": [],
   "source": [
    "disagreement = np.where(class_c == class_py, 0, class_py)\n",
    "\n",
    "fig, axes = plt.subplots(2, 2, figsize=(14, 12), constrained_layout=True)\n",
    "\n",
    "l1 = axes[0, 0].imshow(labels_np[::8], cmap=\"tab10\", interpolation=\"nearest\", vmin=0, vmax = np.max(labels_np))\n",
    "axes[0, 0].set_title(\"Training labels\")\n",
    "axes[0, 0].axis(\"off\")\n",
    "fig.colorbar(l1, ax=axes[0, 0], fraction=0.046)\n",
    "\n",
    "l2 =axes[0, 1].imshow(class_c[::8], cmap=\"tab10\", interpolation=\"nearest\", vmin=0, vmax = np.max(labels_np))\n",
    "axes[0, 1].set_title(\"C supervised Wishart\")\n",
    "axes[0, 1].axis(\"off\")\n",
    "fig.colorbar(l2, ax=axes[0, 1], fraction=0.046)\n",
    "\n",
    "l3 = axes[1, 0].imshow(class_py[::8], cmap=\"tab10\", interpolation=\"nearest\", vmin=0, vmax = np.max(labels_np))\n",
    "axes[1, 0].set_title(\"Python supervised Wishart\")\n",
    "axes[1, 0].axis(\"off\")\n",
    "fig.colorbar(l3, ax=axes[1, 0], fraction=0.046)\n",
    "\n",
    "im = axes[1, 1].imshow(disagreement[::8], cmap=\"magma\", interpolation=\"nearest\")\n",
    "axes[1, 1].set_title(\"Disagreement map (0 means match)\")\n",
    "axes[1, 1].axis(\"off\")\n",
    "fig.colorbar(im, ax=axes[1, 1], fraction=0.046)\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db7acde7",
   "metadata": {},
   "source": [
    "Interpretation:\n",
    "\n",
    "If agreement is high but not perfect, that is expected. The goal here is to verify that the\n",
    "Python implementation lands in the same regime as the C executable on the real training setup,\n",
    "not to force bit-identical output.\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "psp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
