\documentclass[notitlepage,reprint,nofootinbib]{revtex4-2}
\usepackage{float}
\usepackage{hyperref}
\usepackage{pgfgantt}

\begin{document}
\title{Multimodal Deep Learning for Predicting Prostate Cancer Recurrence from Histopathology and Multiparametric MRI}
\author{Sebastian Koranda}
\date{\today}
\noaffiliation

\begin{abstract}
This thesis investigates deep learning methods for integrating whole-slide histopathology and multiparametric MR images to predict biochemical recurrence in prostate cancer. The project evaluates multimodal fusion strategies and survival modeling approaches to determine whether combining micro- and macro-scale imaging features improves prognostic accuracy compared to single-modality models. We will use data from task 1 in the CHIMERA challenge\footnote{\url{https://chimera.grand-challenge.org/task-1-prostate-cancer-biochemical-recurrent-prediction/}}.
\end{abstract}

\maketitle

\section*{Background}
Prostate cancer is a leading cause of cancer-related morbidity and mortality among men worldwide, with biochemical recurrence (BCR) occurring in 20-40\% of patients following radical prostatectomy or radiation therapy. Accurate prediction of BCR is crucial for guiding clinical decision-making regarding adjuvant therapy, surveillance intensity, and patient counseling. Traditional prognostic tools such as the Gleason score, PSA levels, and clinical staging provide important risk stratification, but often lack the granularity needed for personalized treatment planning.

Histopathological analysis of H\&E-stained whole-slide images (WSIs) provides detailed insights into tumor morphology at the cellular level, including glandular architecture, nuclear atypia, and stromal characteristics. Concurrently, multiparametric MRI (mpMRI) offers complementary information on tumor location, size, extracapsular extension, and tissue characteristics at the organ level through T2-weighted imaging, diffusion-weighted imaging (DWI), and dynamic contrast-enhanced (DCE) sequences. Integrating these two imaging modalities through multimodal deep learning has the potential to enhance prognostic accuracy by capturing both micro- and macro-scale features of prostate cancer, potentially identifying patterns invisible to human experts or single-modality analysis.

WSIs are typically very large images, often exceeding several gigapixels in size (e.g., 100,000$\times$100,000 pixels), which poses significant computational challenges for direct analysis. Advanced techniques such as patch-based processing, multiple instance learning (MIL), and the use of foundation models like Prov-GigaPath \cite{xu_whole-slide_2024} can help manage this complexity by extracting meaningful feature representations from tiles while maintaining spatial context. These foundation models, pre-trained on millions of histopathology images, can capture generalizable tissue patterns that are relevant for downstream prognostic tasks.

\begin{figure}[H]
\centering
\includegraphics[width=0.9\linewidth]{../figs/1003_1_he_stained.png}
\caption{Example H\&E-stained whole-slide image (WSI) from patient 1003 in the CHIMERA challenge dataset, reconstructed from tile-based feature embeddings. The image was generated using distance-weighted Gaussian blending of 351,874 tiles, with SVD-based dimensionality reduction mapping 1024-dimensional features to RGB color space. The original WSI (168,736$\times$107,168 pixels) was downscaled by a factor of 30 to 5,624$\times$3,572 pixels for visualization.}
\label{fig:he-stained-wsi}
\end{figure}

\section*{Objectives}
The primary objective of this thesis is to investigate whether multimodal deep learning can improve the prediction of biochemical recurrence in prostate cancer compared to single-modality approaches. Specific aims include:
\begin{itemize}
    \item Develop and validate baseline unimodal models using WSI and MRI data independently.
    \item Explore and compare various multimodal fusion strategies (early, intermediate, late, and hybrid fusion) to integrate histopathological and radiological features.
    \item Evaluate the prognostic performance of multimodal models against unimodal baselines using survival analysis metrics.
    \item Identify which fusion strategy yields the best predictive performance for BCR prediction.
    \item Provide insights into the complementary value of micro-scale (WSI) and macro-scale (MRI) imaging features for cancer prognosis.
\end{itemize}

\section*{Material}
This study will utilize data from Task 1 of the CHIMERA (Computational Histopathology and Medical Imaging for Enhanced Risk Assessment) challenge, which provides a comprehensive dataset of prostate cancer patients with matched imaging modalities and clinical outcomes. The dataset includes:
\begin{itemize}
    \item H\&E-stained whole-slide images (WSIs) from radical prostatectomy specimens, digitized at high resolution (typically 0.25-0.5 $\mu$m per pixel)
    \item Multiparametric MRI scans including T2-weighted, diffusion-weighted imaging (DWI), apparent diffusion coefficient (ADC) maps, and dynamic contrast-enhanced (DCE) sequences
    \item Clinical variables including age, pre-treatment PSA levels, Gleason score, pathological staging, and surgical margin status
    \item Follow-up data with time-to-event information for biochemical recurrence (defined as PSA $\geq$ 0.2 ng/mL)
\end{itemize}
The dataset has been de-identified and anonymized in compliance with HIPAA regulations and ethical standards for medical research. All patients provided informed consent for research use of their clinical data and imaging.

\section*{Methodology}
\subsection*{Data Preprocessing}
\textbf{WSI Processing:}
\begin{itemize}
    \item Tile extraction: Segment tissue regions and extract patches (e.g., 224$\times$224 pixels) at appropriate magnification (e.g., 20$\times$ or 40$\times$)
    \item Feature extraction: Utilize a pre-trained foundation model such as Prov-GigaPath \cite{xu_whole-slide_2024} to extract high-dimensional feature embeddings (e.g., 1024-dimensional vectors) for each tile
    \item Aggregation: Apply multiple instance learning (MIL) or attention-based pooling to aggregate tile-level features into slide-level representations
    \item Quality control: Filter out artifacts, blurred regions, and background areas
\end{itemize}

\textbf{MRI Processing:}
\begin{itemize}
    \item Registration: Co-register different MRI sequences (T2w, DWI, ADC, DCE) to ensure spatial alignment
    \item Normalization: Apply intensity normalization to account for scanner variability and improve model generalization
    \item Segmentation: Delineate prostate gland and tumor regions of interest (ROIs) using automatic or semi-automatic methods
    \item Feature extraction: Extract radiomics features or use deep learning encoders (e.g., 3D CNNs or nnU-Net) to obtain MRI feature representations
\end{itemize}

\subsection*{Model Development}

\subsubsection*{Baseline Models}
\begin{itemize}
    \item Develop and evaluate baseline models using single-modality data (e.g., WSI only, MRI only).
    \item Assess the model performance using survival analyses such as Harrell's c-index, the Kaplan-Meier method, and Cox proportional hazards regression model.
\end{itemize}

\subsubsection*{Multimodal Models}
\begin{itemize}
    \item Explore various multimodal fusion strategies to train a multimodal model including both WSIs and MRIs.
        \begin{itemize}
            \item \textbf{Late:} separate WSI and MRI models, combined at output/prediction layer.
            \item \textbf{Intermediate:} fusion of separately processed, latent representations, then process fused representations.
            \item \textbf{Early:} fusion at input level (likely not justified in this case.)
            \item \textbf{Hybrid:} combination of approaches.
        \end{itemize}
\end{itemize}

\subsection*{Model Evaluation}
\begin{itemize}
    \item \textbf{Validation Strategy:} Employ cross-validation or hold-out test set to assess model generalization and prevent overfitting
    \item \textbf{Primary Metrics:}
        \begin{itemize}
            \item Harrell's concordance index (C-index) for overall survival prediction accuracy
            \item Time-dependent AUC at clinically relevant time points (e.g., 1, 3, 5 years)
            \item Integrated Brier score for calibration assessment
        \end{itemize}
    \item \textbf{Secondary Analyses:}
        \begin{itemize}
            \item Kaplan-Meier survival curves stratified by risk groups
            \item Log-rank tests for statistical significance between risk strata
            \item Cox proportional hazards models to assess independent prognostic value
            \item Ablation studies to understand contribution of each modality and fusion component
        \end{itemize}
    \item \textbf{Comparative Analysis:} Systematically compare unimodal models (WSI-only, MRI-only) against multimodal architectures, and evaluate different fusion strategies (early, intermediate, late, hybrid)
    \item \textbf{Clinical Utility:} Assess whether multimodal models provide clinically meaningful improvement in risk stratification beyond existing prognostic tools
\end{itemize}

\begin{figure}[H]
\centering
\resizebox{0.75\linewidth}{!}{\input{architecture.tex}}
\caption{Possible multimodal deep learning architecture for prostate cancer recurrence prediction, combining WSI and MRI encoders with a fusion module and survival prediction head.}
\label{fig:multimodal-architecture}
\end{figure}

\section*{Expected Contributions}
\begin{itemize}
    \item \textbf{Methodological Contribution:} Comprehensive evaluation of multimodal fusion strategies (early, intermediate, late, hybrid) for integrating histopathology and radiology in cancer prognosis, with quantitative comparison of their relative performance
    \item \textbf{Clinical Impact:} Quantitative assessment of whether multimodal deep learning provides statistically significant and clinically meaningful improvement in BCR prediction compared to single-modality approaches and traditional prognostic factors
    \item \textbf{Technical Innovation:} Novel application of foundation models (e.g., Prov-GigaPath) for WSI feature extraction in the context of multimodal survival analysis
    \item \textbf{Reproducible Framework:} Open-source, well-documented pipeline for CHIMERA Task 1 that can serve as a benchmark for future multimodal survival modeling research
    \item \textbf{Interpretability Insights:} Analysis of which imaging features (micro-scale vs. macro-scale) are most informative for BCR prediction and how they interact in multimodal models
    \item \textbf{Generalizability Assessment:} Evaluation of model robustness across patient subgroups and discussion of potential clinical deployment considerations
\end{itemize}

\newpage
\onecolumngrid
\section*{Progress Plan}
\begin{figure}[H]
\centering
\input{gantt_chart.tex}
\caption{Gantt chart for the planned progress of the master project.}
\label{fig:gantt}
\end{figure}

\newpage
\onecolumngrid
\section*{References}
\nocite{*}
\bibliography{curriculum}


\end{document}
