\section{Introduction to causal models}\label{app:scms}
The causal models that we consider can be interpreted as causal Bayesian networks or structural causal models. For completeness, we briefly present both modelling frameworks.
This material can largely be found in \cite{pearl2009causality,bongers2021foundations,forre2025mathematical,boeken2025are}.

\subsection{Graphs}
A \emph{directed acyclic graph} (DAG) is a tuple $G = (V, E)$ with $V$ a finite set of vertices and $E\subset V\times V$ a set of directed edges such that there are no directed cycles. An \emph{acyclic directed mixed graph} (ADMG) is a tuple $G = (V, E, L)$ with vertices $V$, directed edges $E$, and bidirected edges $L\subset V\times V$ such that there are no directed cycles.

Given DAG $G$ with path $\pi=a\sus...\sus b$, a \emph{collider} is a vertex $v$ with $...\to v \ot...$ in $\pi$. For sets of vertices $A, B, C\subseteq V$ we say that $A$ and $B$ are \emph{$d$-separated} given $C$, written $A\perp^d_G B \given C$, if for every path $\pi = a \sus ... \sus b$ between every $a\in A$ and $b \in B$, there is a collider on $\pi$ that is not an ancestor of $C$, or if there is a non-collider on $\pi$ in $C$. The sets $A$ and $B$ are \emph{$d$-connected} given $C$ if they are not $d$-separated, written $A\nPerp^d_G B \given C$.

Given a DAG $G$ over $V\cup W$, the \emph{latent projection} of $G$ onto $V$ is the ADMG $G_V$ with vertices $V$, directed edges $a\to b$ if there is a path $a \to w_1 \to ... \to w_n \to b$ in $G$ with $w_i\in W$ for all $i=1, ..., n$ (if any), and bi-directed edges $a\tot b$ if there is a bifurcation $a \ot w_1 \ot ... \ot w_k \to ... \to w_n \to b$ in $G$ with $w_i\in W$ for all $i=1, ..., n$ \citep{verma1993graphical}.


\begin{figure}[!htb]
	\centering
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\begin{tikzpicture}
			\node[var] (F) at (1.5, 0) {$F$};
			\node[var] (A1) at (3, 0) {$A_1$};
			\node[var] (L1) at (4.5, 1.5) {$L_1$};
			\node[var] (A2) at (3, 1.5) {$A_2$};
			\node[var] (A3) at (3, -1.5) {$A_3$};
			\node[var] (Y) at (4.5, 0) {$Y$};
			\node[var] (L2) at (4.5, -1.5) {$L_2$};
			\draw[arr] (F) to (A1);
			\draw[arr] (L1) to (A2);
			\draw[arr] (L1) to (Y);
			\draw[arr] (A2) to (A1);
			\draw[arr] (F) to (A3);
			\draw[arr] (A3) to (L2);
			\draw[arr] (L2) to (Y);
			\draw[arr] (A1) to (Y);
			\draw[arr] (A2) to (Y);
		\end{tikzpicture}
		\caption{Graph $G$}
		\label{fig:full_graph}
	\end{subfigure}
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\begin{tikzpicture}
			\node[var] (F) at (1.5, 0) {$F$};
			\node[var] (A1) at (3, 0) {$A_1$};
			\node[var] (A2) at (3, 1.5) {$A_2$};
			\node[var] (A3) at (3, -1.5) {$A_3$};
			\node[var] (Y) at (4.5, 0) {$Y$};
			\draw[arr] (F) to (A1);
			\draw[biarr,bend left] (A2) to (Y);
			\draw[arr] (A2) to (A1);
			\draw[arr] (F) to (A3);
			\draw[arr] (A3) to (Y);
			\draw[arr] (A1) to (Y);
			\draw[arr] (A2) to (Y);
		\end{tikzpicture}
		\caption{Latent projection of $G$ onto $F, A_1, A_2, A_3, Y$.}
		\label{fig:latent_projection}
	\end{subfigure}
	\caption{Example of DAG $G$ and a latent projection.}
	\label{fig:covariates}
\end{figure}

For an ADMG $G$ over $V$, a set of variables $A_1, ..., A_n \subseteq V$ can be \emph{merged} into a single variable $A \notin V$ by removing from $G$ all vertices $A_i$, adding a vertex $A$, and for every $v\in V\setminus \{A_1, ..., A_n\}$ add the edge $v \to A$ / $v \tot V$ / $A \to v$ if there is an $i \in \{1, ..., n\}$ such that $v \to A_i$ / $v\tot A_i$ / $A_i \to v$ in $G$.

\begin{figure}[!htb]
	\centering
	\begin{tikzpicture}
		\node[var] (F) at (1.5, 0) {$F$};
		\node[var] (A) at (3, 0) {$A$};
		\node[var] (Y) at (4.5, 0) {$Y$};
		\draw[arr] (F) to (A);
		\draw[arr] (A) to (Y);
		\draw[biarr,bend left] (A) to (Y);
	\end{tikzpicture}
	\caption{Merging of variables $A_1, A_2, A_3$ of the graph in Figure \ref{fig:latent_projection} to a single vertex $A$.}
	\label{fig:merge}
\end{figure}

In Figure \ref{fig:full_graph}, we see that $F\Perp^d_G Y\given A_1, A_2, A_3$, so defining $A := (A_1, A_2, A_3)$ we indeed have forecast invariance of $Y|A$ (see Theorem \ref{thm:forecastable_iff_separating}). Note that testing for forecast-invariance in the original graph (where the set of variables $A$ is not merged) is stronger than testing it after the merging of $A$: we have $F\Perp^d Y\given A_1, A_2, A_3$ in Figure \ref{fig:full_graph} and equivalently in Figure \ref{fig:latent_projection}, but we have $F\nPerp^d Y\given A$ in Figure \ref{fig:merge}.

\subsection{Bayesian network}
Given a DAG $G$, writing $\pa(v)$ for the set of parents of $v$ in $G$, a \emph{Bayesian network over $G$} is a tuple of Markov kernels $(P(X_v\given X_{\pa(v)}))_{v\in V}$.
We refer to the joint distribution $P(X_V) = \bigotimes_{v\in V} P(X_v \given X_{\pa(v)})$ as the \emph{observational distribution}.

Given a Bayesian network and a subset of variables $T\subseteq V$ and values $x_T \in \Xcal_T$, the intervened Bayesian network is given by the set of Markov kernels
\begin{equation*}
	\tilde{P}(X_v\given X_{\pa(v)}) = \begin{cases}
		\delta_{x_v}(X_v)       & \text{ if $v\in T$}     \\
		P(X_v\given X_{\pa(v)}) & \text{if $v \notin T$}.
	\end{cases}
\end{equation*}
This gives rise to the interventional distribution $P(X_V \given \Do(X_T = x_t)) := \bigotimes_{v\in V} \tilde{P}(X_v \given X_{\pa(v)})$, and the graph of the intervened Bayesian network is the graph $G$ without any edges into any of the variables $v\in T$.
In the main paper we slightly deviate from this notation by writing e.g.\ $P_M(Y\given \Do(F))$ to mean $P_M(X_Y\given \Do(X_F = x_F))$, so only for $F$ we make the exception to always indicate a value $x_F$ instead of the random variable $X_F$.


\subsection{Structural Causal Models}
Alternatively, the causal models can be interpreted as a \emph{structural causal model} (SCM), which formally consist of a tuple $(V, W, \Xcal, f, P)$ where
\begin{itemize}
	% \setlength{\itemsep}{-4pt}
	\item $V$ is the index set of endogenous variables
	\item $W$ is the index set of exogenous variables
	\item $\Xcal = \prod_{v\in V} \Xcal_v \times \prod_{w\in W}\Xcal_w$ is a product of Polish spaces (e.g.\ $\{0,1\}, \ZZ, \RR$)
	\item $f_V: \Xcal_V \times \Xcal_W \to \Xcal_V$ is a set of \emph{structural equations}
	\item $P(X_W) = \bigotimes_{w\in W}P(X_w)$ is a product of Borel probability measures.
\end{itemize}

We call $j\in V\cup W$ a \emph{parent} of $i\in V$ if the structural equation $f_i$ truly depends on $x_j$, i.e.\ if there is no measurable map $\tilde{f}_i : \Xcal_{V\setminus \{j\}}\times \Xcal_{W\setminus \{j\}} \to \Xcal_i$ such that for $P(X_W)$-almost all $x_W$ and all $x_V$ we have
\begin{equation*}
	x_i = f_i(x_{V}, x_W) \iff x_i = \tilde{f}_i(x_{V\setminus \{j\}}, x_{W\setminus \{j\}}).
\end{equation*}

The \emph{augmented graph} of an SCM $M$ is a directed graph $G^a(M)$ with vertices $V\cup W$ and for all $j \in V\cup W, i \in V$ an edge $j\to i$ if $j$ is a parent of $i$. The \emph{graph} $G(M)$ of an SCM $M$ is the latent projection of $G^a(M)$ onto $V$. We call an SCM $M$ \emph{acyclic} if $G^a(M)$ (or equivalently, $G(M)$) is acyclic.

Given an SCM $M = (V, W, \Xcal, f, P)$ and a subset of variables $T\subseteq V$ and value $x_T \in \Xcal_T$, the intervened SCM $M_{\Do(X_T=x_T)} = (V, W, \Xcal, \tilde{f}, P)$ has for every $v\in V$ the structural equation
\begin{equation*}
	\tilde{f}_v = \begin{cases}
		x_v           & \text{ if $v\in T$}     \\
		f_v(x_V, x_W) & \text{if $v \notin T$}.
	\end{cases}
\end{equation*}

Given a subset $T\subseteq V$ and writing $O := V\setminus T$, a \emph{solution function with respect to $O$} is a measurable map $g_O:\Xcal_{T} \times \Xcal_W\to \Xcal_O$ such that $P(X_W)$-almost all $x_W \in \Xcal_W$ and for all $x_{T} \in \Xcal_{T}$ we have $g_O(x_{T}, x_W) = f_O(x_{T}, g_O(x_{T}, x_W), x_W)$. Given a random variable $X_W$ with distribution $P(X_W)$, the random variable $X_V := g_V(X_W)$ defines the \emph{observational distribution} $P_M(X_V)$, and the random variable $X_O := g_O(x_T, X_W)$ defines the \emph{interventional distribution} $P_M(X_O\given \Do(X_T=x_T))$. If $M$ is acyclic, $g_O$ can always be obtained by recursive substitution of the structural equations of $M_{\Do(X_T=x_T)}$.


\subsection{Markov property and faithfulness}
For any Bayesian network or acyclic SCM with DAG $G$ and observational distribution $P$ the \emph{global Markov property} holds:
\begin{equation*}
	A\perp^d_G B \given C \implies X_A \Indep_P X_B \given X_C
\end{equation*}
for all $A, B, C\subseteq V$. A Bayesian network is called \emph{faithful} if for all $A, B, C\subseteq V$ we have
\begin{equation*}
	A\nPerp^d_G B \given C \implies X_A \nIndep_P X_B \given X_C,
\end{equation*}
see also \cite{boeken2025are}.

We refer to a distribution $P(X_V)$ as \emph{compatible} with ADMG $G$ with vertices $V$ if the Markov property holds.





\section{Proofs}\label{app:proofs}

\subsection{Section \ref{sec:correct_forecasts}}

\forecastableiffseparating*
\begin{proof}
	iv) $\implies$ iii): If $F\Perp^d_G Y\given A$ then for every $M\in\Mcal'_G$ we have $P_M(Y\given A, \Do(F)) = P_M(Y\given A)$ by the third rule of the do-calculus \citep{pearl2009causality}.\\
	iii) $\implies$ ii): If $P_M(Y\given A, \Do(F)) = P_M(Y\given A)$ for every $M\in\Mcal'_G$, then $P_M(Y\given A)$ is a correct forecast.\\
	ii) $\implies$ i): This follows immediately from Definition \ref{def:correct_conditional_forecast}.\\
	i) $\implies$ iv): We prove the contrapositive. If $F\nPerp^d_G Y\given A$, then there must be a path $F \to V_1 \to ... \to V_n \to Y$ in $G$ with $V_i \notin A$ for all $i=1, ..., n$. Let $P_0, P_1 \in \Pcal_\Ycal$ with $P_0\neq P_1$. Set
	\begin{align*}
		P_M(V_1|\Do(F))             & = \begin{cases}
			                                \delta_0 & \text{if $F(Y\given A=a) = P_1$ for some $a\in \Acal$} \\
			                                \delta_1 & \text{otherwise}
		                                \end{cases} \\
		P_M(V_{i+1}\given \Do(V_i)) & = \begin{cases}
			                                \delta_0 & \text{if $V_i = 0$} \\
			                                \delta_1 & \text{otherwise}
		                                \end{cases} \quad \text{for all $i=1, ..., n-1$}                  \\
		P_M(Y\given V_n)            & =  \begin{cases}
			                                 \delta_0 & \text{if $V_n = 0$} \\
			                                 \delta_1 & \text{otherwise},
		                                 \end{cases}
	\end{align*}
	and pick for every other variable $Z$ (including the variables in $A$) some independent distribution $P_M(Z)$. Then $M$ is Markov with respect to $G$ so $M\in\Mcal'_G$, and $P_M(Y\given A, \Do(F)) = P_1$ iff $P_1 \notin F(Y\given A)$, so there is no observationally correct forecast for $M$.
\end{proof}


\subsection{Section \ref{sec:scoring}}

% \begin{theorem}\label{thm:nonexistence_observationally_proper_deterministic}
% 	Let $\Mcal'$ be the set of mechanisms such that $Y|A$ is forecast invariant  and $P_M(A\given \Do(F))$ is deterministic for all $M\in \Mcal$. If $S$ is a scoring rule with respect to $\Pcal_\Ycal$ such that there are $P_0, P_1\in\Pcal_\Ycal$ and forecast $\tilde{F}\in \Pcal_\Ycal$ with $\tilde{F}\neq P_1$ and $\ol{S}(P_0, P_0) < \ol{S}(\tilde{F}, P_1)$, then $S$ is not observationally proper for predicting $Y|A$ in $\Mcal'$.
% \end{theorem}
% \begin{proof}[Theorem \ref{thm:nonexistence_observationally_proper_deterministic}]
% 	Let $M$ have $P_M(Y\given \Do(A=1)) = P_1$ and $P_M(Y\given \Do(A\neq 1)) = P_0$ and
% 	\begin{equation*}
% 		P_{M}(A\given \Do(F)) = \begin{cases}
% 			\delta_{1} & \text{if $F(Y \given A=1) = \tilde{F}$} \\
% 			\delta_{0} & \text{otherwise},
% 		\end{cases}
% 	\end{equation*}
% 	then $Y|A$ is forecast-invariant in $M$ with $F^*(Y\given A=1) = P_1$ and $F^*(Y\given A\neq 1) = P_0$ the correct forecast, and the incorrect forecast $\tilde{F}(Y\given A=1) = \tilde{F}$ and $\tilde{F}(Y\given A\neq 1) = P_0$ has
% 	\begin{align*}
% 		\ol{S}(F^*, M) = \ol{S}(P_0, P_0) < \ol{S}(\tilde{F}, P_1) = \ol{S}(\tilde{F}, M),
% 	\end{align*}
% 	so $S$ is not proper with respect to $M \in \Mcal'$.
% \end{proof}

\nonexistenceobservationallyproper*
\begin{proof}
	Let $P_0, P_1 \in \Pcal_\Ycal$ with $P_0\neq P_1$, and let $M$ be such that $P_M(Y\given \Do(A=1)) = P_1$ and $P_M(Y\given \Do(A\neq 1)) = P_0$. For the case where $P_M(A\given \Do(F))$ is deterministic, let
	\begin{equation*}
		P_{M}(A\given \Do(F)) = \begin{cases}
			\delta_{1} & \text{if $F(Y \given A=1) = \tilde{F}$} \\
			\delta_{0} & \text{otherwise},
		\end{cases}
	\end{equation*}
	then $Y|A$ is forecast-invariant in $M$ with $F^*(Y\given A=1) = P_1$ and $F^*(Y\given A\neq 1) = P_0$ the correct forecast, and the incorrect forecast $\tilde{F}(Y\given A=1) = \tilde{F}$ and $\tilde{F}(Y\given A\neq 1) = P_0$ has
	\begin{align*}
		\ol{S}(F^*, M) = \ol{S}(P_0, P_0) < \ol{S}(\tilde{F}, P_1) = \ol{S}(\tilde{F}, M),
	\end{align*}
	so $S$ is not proper with respect to $M \in \Mcal'$.

	For the case where $P_M(A\given \Do(F))$ has full support, let $Q(A)$ be a probability distribution on $\Acal$ with full support and such that $Q(A=1) < \frac{\ol{S}(\tilde{F}, P_1) - \ol{S}(P_0, P_0)}{\ol{S}(P_1, P_1) - \ol{S}(\tilde{F}, P_1)}$; note that this right-hand side is strictly positive by assumption. Then the mechanism
	\begin{equation*}
		P_{M}(A\given \Do(F)) = \begin{cases}
			\frac{1}{2}\cdot Q +  \frac{1}{2}\cdot\delta_{1} & \text{if $F(Y \given A=1) = \tilde{F}$} \\
			\frac{1}{2}\cdot Q +  \frac{1}{2}\cdot\delta_{0} & \text{otherwise}
		\end{cases}
	\end{equation*}
	has full support for all forecasts $F$ and $Y|A$ is forecast-invariant in $M$, and we have correct forecast $F^*(Y\given A=1) = P_1$ and $F^*(Y\given A\neq 1) = P_0$ and observationally incorrect forecast $\tilde{F}(Y\given A=1) = \tilde{F}$ and $\tilde{F}(Y\given A\neq 1) = P_0$, and we have expected scores
	\begin{align*}
		2\ol{S}(F^*, M) & = (Q(A\neq 1) + 1) \ol{S}(P_0, P_0) + Q(A= 1)\ol{S}(P_1, P_1)                                \\
		                & < Q(A\neq 1) \ol{S}(P_0, P_0) + (Q(A= 1) + 1)\ol{S}(\tilde{F}, P_1) = 2\ol{S}(\tilde{F}, M),
	\end{align*}
	so $S$ is not observationally proper.
\end{proof}


\subsection{Section \ref{sec:bdt}}

\utilityscoreproper*
\begin{proof}
	Since $\ol{S}(F, M) = \int U(a_F, y)P_M(\diff y\given \Do(F)) = \int U(a, y)P_M(\diff a, \diff y\given \Do(F))$ we automatically have
	\begin{equation*}
		\argmax_F \ol{S}(F, M) = \argmax_F \int U(a, y)P_M(\diff a, \diff y\given \Do(F)),
	\end{equation*}
	so $S$ is incentive compatible with $U$.
	If $Y|A$ is forecast-invariant, for every $M \in \Mcal'$ the forecast $F(Y \given A) = P_M(Y\given A)$ is a correct forecast. By definition of the Bayes act $a_F$ we have $a_{P_M(Y|A)} = \argmax_a \int U(a, y)P_M(\diff y\given A=a)$ so for $F\in\Pcal_{\Acal\to\Ycal}$ we have
	\begin{align*}
		\ol{S}(F(Y\given A), M) & = \int U(a_F, y)P_M(\diff y\given A=a_F)                                         \\
		                        & \leq \int U(a_{P_M}, y)P_M(\diff y\given A=a_{P_M}) = \ol{S}(P_M(Y\given A), M).
	\end{align*}
	Since the expected score is maximised at a correct forecast we have that $S$ is proper.\\
	If $Y|A$ is not forecast-invariant in $\Mcal_G'$, then by Theorem \ref{thm:forecastable_iff_separating} there does not exist an observationally correct forecast, so the utility score is not proper. (Note that in the proof of Theorem \ref{thm:forecastable_iff_separating}, the counterexample for which no observationally correct forecast exists can easily be altered to let $A=a_F$ be the Bayes act, so that the theorem holds in the class $\Mcal_G'\subseteq \Mcal_{\mathrm{dt}}$.)
\end{proof}

\utilityscorestrictlyproper*
\begin{proof}
	We have
	\begin{equation*}
		\ol{S}(F, M) = \int U(a, y)P_M(\diff a, \diff y\given \Do(F)) + \Delta \ol{S}'(F, M),
	\end{equation*}
	so if $F^* = \argmax_F \int U(a, y)P_M(\diff a, \diff y\given \Do(F))$, then for any other $F$ we have
	\begin{align*}
		\ol{S}(F^*, M) - \ol{S}(F, M) & = \int U(a, y)P_M(\diff a, \diff y\given \Do(F^*))  - \int U(a, y)P_M(\diff a, \diff y\given \Do(F)) \\
		                              & + \Delta(\ol{S}'(F^*, M) - \ol{S}'(F, M))                                                            \\
		                              & > \Delta + \Delta(\ol{S}'(F^*, M) - \ol{S}'(F, M)) \geq 0
	\end{align*}
	since $S' \in [0,1]$, hence
	\begin{equation*}
		\argmax_F \ol{S}(F, M) = \argmax_F \int U(a, y)P_M(\diff a, \diff y\given \Do(F)),
	\end{equation*}
	so $S$ is incentive compatible with $U$.
	If $Y|A$ is forecast-invariant, then for every $M \in \Mcal'$ the forecast $F^*(Y \given A) = P_M(Y\given A)$ is a correct forecast.
	Note that any optimiser $F$ of $\ol{S}(F, M)$ must induce the action $a_F = a^* := a_{F^*}$. If $F(Y\given A=a^*)\neq F^*(Y\given A=a^*)$, then $\ol{S}'(F, M) < \ol{S}'(F^*, M)$ so also $\ol{S}(F, M) < \ol{S}(F', M)$, hence $S$ is observationally strictly proper.
	% Let $a^* = \argmax_a E_M[U(a, Y)\given a]$, then for any other $a\neq a^*$ we have $E_M[U(a^*, Y)\given a^*] - E_M[U(a, Y)\given a] > \Delta$, and since $S' \in [0,1]$ we have for any $F\in\Pcal_{\Acal\to\Ycal}$ that $\Delta(E_M[S'(F(Y\given A=a^*), Y)\given a^*] - E_M[S'(F(Y\given A=a), Y)\given a]) \geq -\Delta$, so
	% \begin{align*}
	% 	E_M[S(F, a^*, Y) & \given a^*] - E_M[S(F, a, Y) \given a]                                                                        \\
	% 	                 & = E_M[U(a^*, Y)\given a^*] - E_M[U(a, Y)\given a]                                                             \\
	% 	                 & ~~~~~+ \Delta\left(E_M[S'(F(Y\given A=a^*), Y)\given a^*] - E_M[S'(F(Y\given A=a), Y)\given a]\right) \geq 0.
	% \end{align*}
	% Since any $F$ for which $a_F = a^*$ optimizes both $\ol{S}(F, M) = E_M[S(F, A, Y) \given \Do(F)]$ and $E_M[U(A, Y)\given \Do(F)]$ we have incentive compatibility. Since any optimizer $F$ of the decision score $S$ induces the action $a_F = a^*$ we have that $F$ optimizes the strictly proper expected score $E_M[S'(F(Y\given A=a^*), Y)\given a^*]$, so $S$ is strictly proper for $F(Y\given A=a^*)$, i.e.\ observationally strictly proper.
\end{proof}


\subsection{Section \ref{sec:divergence}}

\divergenceproper*
\begin{proof}
	If $S$ is proper in the classical sense, we have for every $F(Y\given A)\in\Pcal_{\Acal \to \Ycal}$ and every $a\in\Acal$ that
	\begin{align*}
		\ol{S}(F(Y\given A=a), P_M(Y\given A=a, \Do(F))) & \leq \ol{S}(P_M(Y\given A=a, \Do(F)), P_M(Y\given A=a, \Do(F))) \\
		                                                 & =: H(P_M(Y\given A=a, \Do(F)))
	\end{align*}
	so the divergence is positive:
	\begin{align}
		\begin{split}
			\label{eq:divergence}
			& D_{\mathrm{pc}}(F, M) = \int \left(S(P_M(Y\given A=a, \Do(F)), y) - S(F(Y\given A=a), y)\right)P_M(\diff a, \diff y\given \Do(F)) \\
			& = \int \left(H(P_M(Y\given A=a, \Do(F))) - \ol{S}(F(Y\given A=a), P_M(Y\given A=a, \Do(F)))\right)P_M(\diff a\given \Do(F))\\
			&\geq 0.
		\end{split}
	\end{align}
	If $F^*(Y\given A)$ is correct for $M$, then $F^*(Y\given A=a) = P_M(Y\given A=a, \Do(F^*))$ for all $a\in\Acal$ such that $P_M(A=a\given \Do(F^*)) > 0$, so for every such $a$ we have
	\begin{align*}
		\ol{S}(F^*(Y\given A=a), P_M(Y\given A=a, \Do(F^*))) & = \ol{S}(P_M(Y\given A=a, \Do(F^*)), P_M(Y\given A=a, \Do(F^*))) \\
		                                                     & = H(P_M(Y\given A=a, \Do(F)))
	\end{align*}
	and by plugging this into (\ref{eq:divergence}) we get $D_{\mathrm{pc}}(F^*, M) = 0$, so $-D_{\mathrm{pc}}$ is indeed proper.
	If $S$ is strictly proper, then for any $F(Y\given A=a) \neq P_M(Y\given A=a, \Do(F))$ for some $a$ such that $P_M(A=a\given \Do(F)) > 0$ then we have
	\begin{align*}
		H(P_M(Y\given A=a, \Do(F))) - \ol{S}(F(Y\given A=a), P_M(Y\given A=a, \Do(F^*)))  > 0
	\end{align*}
	and hence $D_{\mathrm{pc}}(F(Y\given A), M) > 0$, so $-D$ is indeed strictly observationally proper.
\end{proof}

\unbiasedproper*
\begin{proof}
	By unbiasedness of the estimator we have $\int\hat{D}_{\mathrm{pc}}(A_1, ..., Y_1)\diff P_M^n(A, Y) \given \Do(F) = D_{\mathrm{pc}}(F, M)$, so the result immediately follows from Theorem \ref{thm:divergence_proper}.
\end{proof}

\unbiasedestimators*
\begin{proof}
	Note that we have $D_{\mathrm{pc}}^{Brier}(F, M) = \sum_{a\in\Acal}(p_a - f_a)^2P_M(A=a\given\Do(F))$.
	By taking the expectation over the data, we have
	\begin{equation*}
		\EE\left[\hat{D}_{\mathrm{pc}}^{Brier}\right] = \EE\left[\sum_{a\in\Acal}\frac{n_a}{n}(\hat{p}_{a_i} - F_{a_i})^2 - \frac{\hat{p}_{a_i} (1-\hat{p}_{a_i})}{n_{a_i}-1}\right].
	\end{equation*}
	Writing $p_a := P(Y=1 \given A, \Do(F))$, for a given $a\in\Acal$ in the summation and by conditioning on $n_a$ we have $\EE[(\hat{p}_a - f_a)^2 \given n_a] = (p_a - f_a)^2 + \frac{1}{n_a}p_n(1-p_n)$ and $\EE[\hat{p}_a(1-\hat{p}_a)\given n_a] = \frac{n_a - 1}{n_a}p_a(1-p_a)$, and hence $\EE[\frac{n_a}{n}(\hat{p}_{a_i} - f_{a_i})^2 - \frac{\hat{p}_{a_i} (1-\hat{p}_{a_i})}{n_{a_i}-1} \given n_a] = \frac{n_a}{n}(p_a - f_a)^2$. Integrating out the $n_a$ we have $\EE[\frac{n_a}{n}] = P_M(A\given \Do(F))$ and hence $\EE[\hat{D}_{\mathrm{pc}}^{Brier}] = \sum_{a\in\Acal} (p_a - f_a)^2 P(A\given \Do(F)) = D_{\mathrm{pc}}^{Brier}(F, M)$, which is the desired result.

	For the Energy score, the divergence $D_{\mathrm{pc}}^{Energy}$ is equal to
	\begin{equation*}
		\sum_{a\in\Acal}\left(
		\EE_{\substack{Y\sim F_a\\ Y'\sim P_a}} [\|Y-Y'\|]
		- \frac{1}{2}\EE_{Y,Y'\sim P_a} [\|Y-Y'\|]
		- \frac{1}{2}\EE_{Y, Y'\sim F_a} [\|Y-Y'\|]
		\right)P_M(A=a\given \Do(F)).
	\end{equation*}
	Taking the expectation of the first term in the estimator we obtain $\EE\left[\frac{1}{n}\sum_{i=1}^n \EE_{Y\sim F_{a_i}}\left[\|Y-y_i\|\right]\right] = \sum_{a\in\Acal}\EE_{Y\sim F_{a}, Y'\sim P_a}\left[\|Y-Y'\|\right]P_M(A=a\given \Do(F))$.
	Denoting $I_a := \{i : A_i = a\}$, we see for the second term that
	\begin{equation*}
		\frac{1}{n}\sum_{i=1}^n\frac{1}{2}\frac{\sum_{\substack{j \neq i \\ a_j = a_i}}\|y_i - y_j\|}{n_{a_i}-1} = \sum_{a\in\Acal}\frac{1}{2}\left(\frac{1}{n_{a}(n_{a}-1)}\sum_{\substack{i\neq j\\i,j\in I_a}}\|Y_i - Y_j\|\right)\frac{n_a}{n},
	\end{equation*}
	where the part between brackets is a standard U-statistic for the norm (also known as \emph{Gini's mean difference}) of independent random variables $Y, Y'\sim P_M(Y\given A=a, \Do(F))$, so the expectation of this term equals $\sum_{a\in\Acal}\frac{1}{2}\EE_{Y,Y'\sim P_a} [\|Y-Y'\|]P_M(A=a\given \Do(F))$ \citep{szekely2013energy}.
	The expected value of the last term clearly equals $\sum_{a\in \Acal}\frac{1}{2}\EE_{Y, Y'\sim F_a} [\|Y-Y'\|]P_M(A=a\given \Do(F))$, so upon combining these results we see that $\EE[\hat{D}_{\mathrm{pc}}^{Energy}] = D_{\mathrm{pc}}^{Energy}$.
\end{proof}


\subsection{Section \ref{sec:estimation}}
\performativestability*
\begin{proof}
	Since $P_M(A\given \Do(F_{\theta_t}))$ has full support, we have by Theorem \ref{thm:conditional_divergence_proper} that the decoupled performative risk $R^{\mathrm{d}}_D$ is strictly proper, hence we have that $F_{\theta_{t+1}}$ is correct. Similarly we have that $\theta_{t+2} := \argmin_\theta R^{\mathrm{d}}_D(\theta, \theta_{t+1})$ is correct as well, hence $\theta_{t+1} = \theta_{t+2}$, so $\theta_{t+1}$ is performatively stable. By Theorem \ref{thm:divergence_proper} we have that the performative risk $R_D$ is strictly proper as well, so $R_D(\theta_{t+1})=0$, hence $\theta_{t+1}$ is performatively optimal.
\end{proof}







\section{Proper scoring rules for non-performative conditional forecasts}\label{app:conditional_forecasts_proper}
As a special case of Definition \ref{def:correct_conditional_forecast}, given a Markov kernel $P(Y\given A)\in \Pcal_{\Acal\to \Ycal}$ and distribution $P(A) \in \Pcal_{\Acal}$, the conditional forecast $F(Y\given A)\in\Pcal_{\Acal\to\Ycal}$ is:
\begin{itemize}
	\setlength{\itemsep}{-1pt}
	\item \emph{observationally correct} if $F(Y\given A=a) = P(Y\given A=a)$ for all $a\in\Acal$ such that $P(A=a) > 0$;
	\item \emph{counterfactually correct} if $F(Y\given A=a) = P(Y\given A=a)$ for all $a\in\Acal$ such that $P(A=a) = 0$;
	\item \emph{correct} if it is observationally and counterfactually correct.
\end{itemize}

Given a conditional scoring rule $S:\Pcal_{\Acal \to \Ycal} \times \Acal \to \Ycal \to \RR$, conditional forecast $F(Y\given A) \in \Pcal_{\Acal \to \Ycal}$ and model $(P(Y\given A), P(A)) \in \Pcal_{\Acal \to \Ycal}\times \Pcal_\Acal$, the expected score is
\begin{equation*}
	\ol{S}_{\mathrm{c}}(F(Y\given A), P(A, Y)) := \int S(F(Y\given A), a, y)P(\diff a, \diff y),
\end{equation*}
where $P(A, Y) := P(Y\given A)\otimes P(A)$.

\begin{definition}\label{def:proper_conditional}
	We call a conditional scoring rule
	\begin{itemize}
		\setlength{\itemsep}{-4pt}
		\item \emph{observationally (strictly) proper} relative to $\Pcal_{\Acal \to \Ycal} \times \Pcal_\Acal$ if $\ol{S}_{\mathrm{c}}$ is (only) maximised at observationally correct forecasts for all $M\in \Mcal'$;
		\item \emph{counterfactually (strictly) proper} relative to $\Pcal_{\Acal \to \Ycal}\times \Pcal_\Acal$ if $\ol{S}_{\mathrm{c}}$ is (only) maximised at counterfactually correct forecasts for all $M\in \Mcal'$;
		\item \emph{(strictly) proper} relative to $\Pcal_{\Acal \to \Ycal}\times \Pcal_\Acal$ if it is observationally and counterfactually (strictly) proper, that is, if $\ol{S}_{\mathrm{c}}$ is (only) maximised at correct forecasts for all $(P(Y\given A), P(A)) \in \Pcal_{\Acal \to \Ycal}\times \Pcal_\Acal$.
	\end{itemize}
\end{definition}

Recall that any scoring rule $S:\Pcal_\Ycal \times \Ycal\to \RR$ induces a conditional scoring rule $S'$ through $S'(F, a, y) := S(F(Y\given A=a), y)$.
\begin{theorem}\label{thm:marginal_conditional_proper}
	If $S$ is a proper scoring rule for marginal forecasts with respect to $\Pcal_\Ycal$, then the induced conditional scoring rule is proper with respect to $\Pcal_{\Acal \to \Ycal}\times \Pcal_\Acal$. If $S$ is strictly proper, then the induced conditional scoring rule is observationally strictly proper.
\end{theorem}
\begin{proof}
	Given $(P(Y\given A), P(A)) \in \Pcal_{\Acal \to \Ycal}\times \Pcal_\Acal$ we have for every $a\in \Acal$ with $P(A=a)>0$ that $F(Y\given A=a), P(Y\given A=a) \in \Pcal_\Ycal$, and since $S$ is proper in the classical sense we have
	\begin{equation}\label{eqn:conditional}
		\ol{S}(F(Y\given A=a), P(Y\given A=a)) \leq \ol{S}(P(Y\given A=a), P(Y\given A=a)).
	\end{equation}
	Integrating with respect to $P(A)$ gives
	\begin{align}
		\begin{split}\label{eqn:integrated}
			\ol{S}_c(F(Y\given A), P(A,Y)) & = \int \ol{S}(F(Y\given A=a), P(Y\given A=a))P(\diff a)                                            \\
			& \leq \int \ol{S}(P(Y\given A=a), P(Y\given A=a)) P(\diff a) = \ol{S}_c(P(Y\given A), P),
		\end{split}
	\end{align}
	and since $P(Y\given A)$ is correct, $S$ is proper.
	If $S$ is strictly proper in the classical sense, then for $F(Y\given A) \neq P(Y\given A)$ there is an $a$ with $P(A=a)>0$ such that $F(Y\given A=a) \neq P(Y\given A=a)$, in which case we have a strict inequality in (\ref{eqn:conditional}), giving a strict inequality in (\ref{eqn:integrated}), so $S$ is observationally strictly proper. Note that any other $Q(Y\given A)\in \Pcal_{\Acal \to \Ycal}$ such that $Q(Y\given A=a) = P(Y\given A=a)$ for all $a\in\Acal$ with $P(A=a)>0$ obtains the same expected score but need not be counterfactually correct, so $S$ is not counterfactually strictly proper.
\end{proof}

\subsection{Divergence for non-performative conditional forecasts}
For a conditional forecast $F(Y\given A) \in \Pcal_{\Acal \to \Ycal}$ and model $(P(Y\given A), P(A)) \in \Pcal_{\Acal \to \Ycal}\times \Pcal_\Acal$, we define the generalised conditional entropy and generalised conditional divergence as
\begin{align*}
	H_{\mathrm{c}}(P(A, Y))               & := \int H(P(Y \given A=a))P(\diff a)                                    \\
	D_{\mathrm{c}}(F(Y\given A), P(A, Y)) & := H_{\mathrm{c}}(P) - \ol{S}_{\mathrm{c}}(F(Y\given A), P)             \\
	                                      & = \int S(P(Y \given A=a), y) - S(F(Y\given A=a), y)P(\diff a, \diff y).
\end{align*}
For the log-score, the entropy $H_{\mathrm{c}}$ is also known as the \emph{conditional entropy} \citep{cover2006elements}.

For the divergence, we obtain a theorem similar to Theorem \ref{thm:marginal_conditional_proper}, that the generalised conditional divergence is observationally (strictly) proper if it is induced by a (strictly) proper scoring rule.
\begin{theorem}\label{thm:conditional_divergence_proper}
	If $S$ is a proper scoring rule, then $D_{\mathrm{c}}$ is proper for conditional forecasts. If $S$ is strictly proper, then $D_{\mathrm{c}}$ observationally strictly proper.
\end{theorem}
\begin{proof}
	Given $(P(Y\given A), P(A)) \in \Pcal_{\Acal \to \Ycal}\times \Pcal_\Acal$ and proper scoring rule $S$ we have
	\begin{equation}
		D_{\mathrm{c}}(F(Y\given A), P(A, Y)) = \int D(F(Y\given A=a), P(Y\given A=a))P(\diff a) \geq 0
	\end{equation}
	and $D_{\mathrm{c}}(F(Y\given A), P(A, Y)) = 0$, so $D_{\mathrm{c}}$ is proper. If $S$ is strictly proper then for every $a\in\Acal$ with $P(A=a)>0$ we have $D(F(Y\given A=a), P(Y\given A=a))=0$ if and only if $F(Y\given A=a) = P(Y\given A=a)$, hence $D_{\mathrm{c}}(F, P) = 0$ only if $F$ is observationally correct.
	% Integrating with respect to $P(A)$ gives
	% \begin{align}
	% 	\begin{split}\label{eqn:integrated}
	% 		\ol{S}_c(F(Y\given A), P) & = \int \ol{S}(F(Y\given A=a), P(Y\given A=a))P(\diff a)                                            \\
	% 		& \leq \int \ol{S}(P(Y\given A=a), P(Y\given A=a)) P(\diff a) = \ol{S}_c(P(Y\given A), P).
	% 	\end{split}
	% \end{align}
	% If $S$ is strictly proper in the classical sense, then for $F(Y\given A) \neq P(Y\given A)$ there is an $a$ with $P(A=a)>0$ such that $F(Y\given A=a) \neq P(Y\given A=a)$, in which case we have a strict inequality in (\ref{eqn:conditional}), giving a strict inequality in (\ref{eqn:integrated}).
\end{proof}





\section{Proper scoring with inverse probability weights}\label{app:ipw}
By scoring with divergence, the utilised scoring rule is made performatively proper by subtracting the (estimated) entropy. As alternative approach, one can multiply the score by (estimates of) the \emph{inverse probability weights} (IPWs) $1 / P_M(A\given \Do(F))$, and consider the expected \emph{IPW score}:
\begin{align*}
	\ol{S}_{\mathrm{IPW}}(F, M) & := \int \frac{S(F(Y\given A=a), y)}{P_M(A=a\given \Do(F))}P(\diff a, \diff y \given \Do(F)) \\
	                            & = \sum_{a\in\Acal}\int S(F(Y\given A=a), y)P(\diff y \given a).
\end{align*}
\cite{chen2011decision} introduced the IPW score in settings where the principal knows $P_M(A)$ (which does not depend on $F$), who can then after every observation $(a,y)$ properly score with $S_{\mathrm{IPW}}(F, a, y) := S(F(Y\given A=a), y) / P_M(A=a)$.
If the distribution $P_M(A\given \Do(F))$ has full support, then this procedure is strictly proper:
\begin{theorem}\label{thm:ipw_score_proper}
	Suppose that for every model $M$ in the class $\Mcal'\subseteq \Mcal_{\mathrm{pc}}$ and every forecast $F\in\Pcal_{\Acal \to \Ycal}$ the distribution $P_M(A\given \Do(F))$ has full support, and that $Y|A$ is forecast-invariant in $M\in\Mcal'$. If $S$ is (strictly) proper in the classical sense, then $\ol{S}_{\mathrm{IPW}}$ is (strictly) proper in the performative sense.
\end{theorem}
\begin{proof}
	By forecast-invariance of $Y|A$ we have
	\begin{align*}
		\ol{S}_{\mathrm{IPW}}(F, M) & = \int \int S(F(Y\given A=a), y)P_M(\diff y\given A=a)\diff a \\
		                            & = \int S(F(Y\given A=a), P_M(Y\given A=a))\diff a,
	\end{align*}
	so for every $a\in\Acal$ the optimum of the integrand lies at the correct forecast $F^*(Y\given A=a) = P_M(Y\given A=a)$, and hence $P_M(Y\given A)$ maximises $S_{\mathrm{IPW}}$, so $\ol{S}_{\mathrm{IPW}}$ is proper. If $S$ is strictly proper, then for every $F$ with $F(Y\given A=a)\neq F^*(Y\given A=a)$ for some $a\in\Acal$ we have $S(F(Y\given A=a), P_M(Y\given A=a)) < S(F^*(Y\given A=a), P_M(Y\given A=a))$, so $\ol{S}_{\mathrm{IPW}}(F, M) < \ol{S}_{\mathrm{IPW}}(F^*, M)$, hence $\ol{S}_{\mathrm{IPW}}$ is strictly proper.
\end{proof}
In Section \ref{app:examples_divergence_ipw} we provide an example where $\ol{S}_{\mathrm{IPW}}$ is not counterfactually proper when $P_M(A\given \Do(F))$ does not have full support.

A similar principle as for the divergence applies: if the principal does not have access to $P_M(A\given \Do(F))$, an unbiased estimate of the mean IPW score suffices for the method to be (observationally strictly) proper. It is unclear whether unbiased estimators for the mean IPW score exist. In Section \ref{app:experiments_divergence_ipw} we provide a brief empirical investigation into the consistency of the plugin estimator for the mean IPW score.
% \begin{corollary}
% 	Suppose that for every model $M$ in the class $\Mcal'\subseteq \Mcal_{\mathrm{pc}}$ and every forecast $F\in\Pcal_{\Acal \to \Ycal}$ the distribution $P_M(A\given \Do(F))$ has full support, and that $Y|A$ is forecast-invariant in $M\in\Mcal'$, and let $S$ be (strictly) proper in the classical sense. Let $(A_1, Y_1), ..., (A_n, Y_n) \sim P_M(A, Y\given \Do(F))$. Any unbiased estimator $\hat{S}_{\mathrm{IPW}}(A_1, ..., Y_n)$ of the mean IPW score $\ol{S}_{\mathrm{IPW}}$ is (strictly) proper, for any sample size $n\in\NN$ such that $\hat{S}_{\mathrm{IPW}}(A_1, ..., Y_n)$ is well-defined.
% \end{corollary}


\section{Forecasts depending on covariates}\label{app:covariates}
\subsection{Causal model including covariates}
We extend the results from the main paper to the setting where we let the forecast $F$ depend explicitly on covariates $X$. In the main paper, we use the notation $P_M(Y\given \Do(F))$ such that for every forecast $F \in \Pcal_{\Ycal}$ we can meaningfully talk about the distribution of $Y$, given the forecast $F$. If we let $F$ depend on covariates $X$, then intervening on $F$ would make the forecast invariant of $X$, which is not what we intend to do. Therefore, we consider a parameter $\theta \in \Theta\subseteq \RR^d$ for some $d$ which specifies a forecasting \emph{model} $F_\theta(Y\given X) \in \Pcal_{\Xcal \to \Ycal}$. In this setting, we will instead of $P_M(Y\given \Do(F))$, for each observed $x\in\Xcal$ consider the causal effect of a specific parameter $\theta$ and its induced forecast $F_\theta(Y\given X=x)$ on $Y$, which we express via $P_M(Y\given \Do(\theta))$, which is equal to $\int P_M(Y\given \Do(\theta, F_\theta(X=x)))P_M(\diff x)$ by consistency.

Formally, for marginal forecasts we let a parameter $\theta$ induce the forecasting model $F_\theta(Y\given X)\in \Pcal_{\Xcal\to \Ycal}$, where for every observed value $x$ and parameter $\theta$ the reported marginal forecast is $F_\theta(Y\given X=x) \in \Pcal_\Ycal$. For this setting let $\Mcal_{\mathrm{p}+}$ be the set of causal models whose projection onto $X,\theta,F,Y$ is a subgraph of Figure \ref{fig:covariates_marginal}.

For conditional forecasts, let $\theta$ induce the forecasting model $F_\theta(Y\given A, X)\in \Pcal_{\Acal\times \Xcal \to \Ycal}$, where for every observed value $x$ and parameter $\theta$ the reported conditional forecast is $F_\theta(Y\given A, X=x)\in \Pcal_{\Acal\to \Ycal}$; let $\Mcal_{\mathrm{pc}+}$ be the set of models whose projection onto $X,\theta,F,A,Y$ is a subgraph of Figure \ref{fig:covariates_conditional}.

\begin{figure}[!htb]
	\centering
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\begin{tikzpicture}
			\node[var] (X) at (0, 0) {$X$};
			\node[var] (F) at (1.5, 0) {$F$};
			\node[var] (Y) at (3, 0) {$Y$};
			\node[var] (theta) at (0.75, 1.25) {$\theta$};
			\draw[arr] (X) to (F);
			\draw[arr, bend right] (X) to (Y);
			\draw[arr] (theta) to (F);
			\draw[biarr, bend left] (X) to (Y);
			\draw[arr] (F) to (Y);
		\end{tikzpicture}
		\caption{$\Mcal_{\mathrm{p}+}$: Marginal forecasts}
		\label{fig:covariates_marginal}
	\end{subfigure}
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\begin{tikzpicture}
			\node[var] (X) at (0, 0) {$X$};
			\node[var] (F) at (1.5, 0) {$F$};
			\node[var] (A) at (3, 0) {$A$};
			\node[var] (Y) at (4.5, 0) {$Y$};
			\node[var] (theta) at (0.75, 1.25) {$\theta$};
			\draw[arr] (X) to (F);
			\draw[arr, bend right] (X) to (A);
			\draw[arr, bend right] (X) to (Y);
			\draw[arr] (theta) to (F);
			\draw[arr] (F) to (A);
			\draw[arr] (A) to (Y);
			\draw[biarr, bend left] (X) to (Y);
			\draw[biarr, bend left] (A) to (Y);
			\draw[arr, bend right] (F) to (Y);
		\end{tikzpicture}
		\caption{$\Mcal_{\mathrm{pc}+}$: Conditional forecasts.}
		\label{fig:covariates_conditional}
	\end{subfigure}
	\caption{Causal graph of performative forecast depending on covariates $X$.}
	% \label{fig:covariates}
\end{figure}

\subsection{Section \ref{sec:correct_forecasts}: Correctness of forecasts}
In the following sections, in each definition or theorem, the corresponding definition/theorem number from the main paper is added in parentheses.
\begin{definition}[Definitions \ref{def:correct_forecast}, \ref{def:correct_conditional_forecast}]
	For a marginal forecast $F_\theta(Y\given X=x)$, the parameter is correct for $M\in \Mcal_{\mathrm{pc}+}$ if $F_\theta(Y\given X=x) = P_M(Y\given X=x, \Do(\theta))$ holds for $P_M(X)$-almost all $x\in\Xcal$. For a conditional forecast $F_\theta(Y\given A, X=x)$, the parameter $\theta$ is
	\begin{itemize}
		\setlength{\itemsep}{-1pt}
		\item \emph{observationally correct} if for $P_M(X)$-almost all $x\in\Xcal$, for all $a\in\Acal$ such that $P(A=a\given X=x, \Do(\theta)) > 0$ we have $F_\theta(Y\given A=a, X=x) = P(Y\given A=a, X=x, \Do(\theta))$;
		\item \emph{counterfactually correct} if for $P_M(X)$-almost all $x\in\Xcal$, for all $a\in\Acal$ such that $P(A=a\given X=x, \Do(\theta)) = 0$ we have $F_\theta(Y\given A=a, X=x) = P(Y\given A=a, X=x, \Do(\theta))$;
		\item \emph{correct} if it is observationally and counterfactually correct.
	\end{itemize}
\end{definition}

\begin{definition}[Definition \ref{def:forecast_invariance}]
	We call the target $Y|A,X$ \emph{forecast-invariant} in $\Mcal'\subseteq \Mcal_{\mathrm{pc}+}$ if the target distribution of $Y|A,X$ does not depend on the forecast, so if $P_M(Y\given A, X, \Do(\theta)) = P_M(Y\given A, X)$ for all $M\in\Mcal'$.
\end{definition}

\begin{theorem}[Theorem \ref{thm:forecastable_iff_separating}]
	Let $\Mcal_G'\subseteq \Mcal_{\mathrm{pc}+}$ be the set of models compatible with causal graph $G$, then the following are equivalent:
	\begin{enumerate}[label=\roman*)]
		% \setlength{\itemsep}{-1pt}
		\item For every $M\in \Mcal'_G$ there exists an observationally correct parameter for $Y|A,X$;
		\item For every $M\in \Mcal'_G$ there exists a correct parameter for $Y|A,X$;
		\item $Y|A,X$ is forecast-invariant for all $M \in \Mcal'_G$;
		\item $\theta\Perp_G^d Y\given A, X$.
	\end{enumerate}
\end{theorem}
\begin{proof}
	The implications iv) $\implies$ iii) $\implies$ ii) $\implies$ i) apply without any change of the proof; for i) $\implies$ iv) one has to also specify an independent distribution $P_M(X)$ to finish the proof.
\end{proof}


\subsection{Section \ref{sec:scoring}: Scoring rules}
Consider a scoring rule $S:\Pcal_{\Ycal}\times \Xcal\times\Ycal \to \RR$, where given a marginal forecast $F_\theta(Y\given X=x)$ and an observed outcome $y$, the forecaster receives the score $S(F_\theta(Y\given X=x), x, y)$. Considering $M\in\Mcal_{\mathrm{pc}+}$, the expected score is
\begin{equation*}
	\ol{S}_{\mathrm{p}+}(F_\theta, M) = \int S(F_\theta(Y\given X=x), x, y)P_M(\diff x, \diff y\given \Do(\theta)).
\end{equation*}
For a conditional forecast $F_\theta(Y\given A, X=x)$, conditional scoring rule $S:\Pcal_{\Acal\to \Ycal}\times \Acal\times\Xcal\times\Ycal \to \RR$, observed action $a$ and outcome $y$, the forecaster receives the conditional score $S(F(Y\given A, X=x), a, x, y)$, and given a model $M\in\Mcal_{\mathrm{pc}+}$ the expected score is
\begin{equation*}
	\ol{S}_{\mathrm{pc}+}(F_\theta, M) = \int S(F_\theta(Y\given X=x), a, x, y)P_M(\diff a,\diff x, \diff y\given \Do(\theta)).
\end{equation*}

\begin{definition}[Definition \ref{def:performative_proper}]
	For marginal forecasts, a scoring rule $S$ is \emph{proper} relative to $\Mcal' \subseteq \Mcal_{\mathrm{p}+}$ if $\ol{S}_{\mathrm{p}+}$ is maximised at a correct forecast, and \emph{strictly proper} if all maximisers are correct. We call a conditional scoring rule
	\vspace*{-5pt}
	\begin{itemize}
		\setlength{\itemsep}{-4pt}
		\item \emph{observationally (strictly) proper} relative to $\Mcal' \subseteq \Mcal_{\mathrm{pc}+}$ if $\ol{S}_{\mathrm{pc}+}$ is (only) maximised at observationally correct forecasts for all $M\in \Mcal'$;
		\item \emph{counterfactually (strictly) proper} relative to $\Mcal' \subseteq \Mcal_{\mathrm{pc}+}$ if $\ol{S}_{\mathrm{pc}+}$ is (only) maximised at counterfactually correct forecasts for all $M\in \Mcal'$;
		\item \emph{(strictly) proper} relative to $\Mcal' \subseteq \Mcal_{\mathrm{pc}+}$ if it is observationally and counterfactually (strictly) proper, that is, if $\ol{S}_{\mathrm{pc}+}$ is (only) maximised at correct forecasts for all $M\in \Mcal'$.
	\end{itemize}
\end{definition}

\begin{theorem}[Theorem \ref{thm:nonexistence_observationally_proper}]
	Let $\Mcal'\subseteq \Mcal_{\mathrm{pc}+}$ be the set of models such that $Y|A,X$ is forecast-invariant, where $P_M(A\given X=x, \Do(\theta))$ has full support for all $\theta\in \Theta$, $P_M(X)$-almost all $x\in\Xcal$ and all $M\in \Mcal'$, or where $P_M(A\given X=x, \Do(\theta))$ is deterministic for all $\theta\in \Theta$, $P_M(X)$-almost all $x\in\Xcal$ and all $M\in \Mcal'$. If $S$ is a scoring rule with respect to $\Pcal_\Ycal$ such that there are $P_0, P_1\in\Pcal_\Ycal$ and forecast $\tilde{F}\in \Pcal_\Ycal$ with $\tilde{F}\neq P_1$ and $\ol{S}(P_0, P_0) < \ol{S}(\tilde{F}, P_1) < \ol{S}(P_1, P_1)$, then $S$ is not observationally proper for predicting $Y|A,X$ in $\Mcal'$.
\end{theorem}
\begin{proof}
	The proof proceeds as in the original proof of Theorem \ref{thm:nonexistence_observationally_proper} with one adjustment: the constructed counterexample specifies a mechanism for $F \to A \to Y$; specifying any independent distribution $P_M(X)$ completes the proof.
\end{proof}


\subsection{Section \ref{sec:bdt}: Decision theory}
Given a conditional forecast $F_\theta$, let the agent take an action which maximizes the expectation of a utility $U(a, x, y)$, so we have $A = f_A(F_\theta,x) := \argmax_{a\in \Acal} \int U(a, x, y)F_\theta(\diff y\given x, a)$. Let $\Mcal_{\mathrm{dt}+}\subseteq \Mcal_{\mathrm{pc}+}$ be the class of models with $P_M(A\given X=x, \Do(\theta)) = \delta_{a_{\theta, x}}$.
\begin{definition}[Definition \ref{def:incentive_compatible}]
	Given a utility $U$ a set of causal models $\Mcal' \subseteq \Mcal_{\mathrm{pc}+}$, we call a conditional scoring rule $S$ \emph{incentive compatible} with $U$ if for all $M\in \Mcal'$ we have $\argmax_\theta\ol{S}_{\mathrm{pc}+}(F_\theta, M) = \argmax_\theta \int U(a, x, y)P_M(\diff a, \diff x, \diff y \given \Do(\theta))$.
\end{definition}
The \emph{utility score} is given by $S(F_\theta, a, x, y) := U(f_A(F_\theta,x), x, y)$.

\begin{theorem}[Theorem \ref{thm:utility_score_proper}]
	The utility score is incentive-compatible with $U$, and proper if $Y|A,X$ is forecast-invariant in $\Mcal' \subseteq \Mcal_{\mathrm{dt}+}$.
	If for a given causal graph $G$ one considers the class of models $\Mcal_G' \subseteq \Mcal_{\mathrm{dt}+}$ which are consistent with $G$, then forecast invariance of $Y|A,X$ is \emph{necessary} for the utility score to be proper.
\end{theorem}
\begin{proof}
	The proof proceeds analogous to the proof of Theorem \ref{thm:utility_score_proper}.
\end{proof}

\begin{theorem}[Theorem \ref{thm:utility_score_strictly_proper}]
	If $Y|A$ is forecast-invariant in $\Mcal' \subseteq \Mcal_{\mathrm{dt}+}$, $U \in [0,1]$, $S' \in [0,1]$ is a proper scoring rule in the classical sense and $\min\left\{\left|E_M(U(a, x, Y)\given a) - E_M(U(a', x, Y)\given a')\right|:a\neq a'\right\} > \Delta > 0$ for $P_M(X)$-almost all $x\in\Xcal$ and all $M\in \Mcal'$, then $S(F, a, x, y) = U(f_A(F_\theta,x), x, y) + \Delta_x\cdot S'(F(Y\given A=f_A(F_\theta,x)), y)$ is proper and incentive-compatible with $U$. If $S'$ is strictly proper in the classical sense, then $S$ is observationally strictly proper.
\end{theorem}
\begin{proof}
	The proof proceeds analogous to the proof of Theorem \ref{thm:utility_score_strictly_proper}.
\end{proof}

\subsection{Section \ref{sec:divergence}: Divergence}
\begin{definition}[Definition \ref{def:performative_divergence}]
	Given a scoring rule $S:\Pcal_\Ycal \times \Ycal\to \RR$, conditional forecast $F_\theta(Y\given A, X=x) \in \Pcal_{\Acal \to \Ycal}$ and causal model $M\in \Mcal_{\mathrm{pc}+}$, the \emph{performative divergence} is defined as
	\begin{equation*}
		D_{\mathrm{pc}+}(F_\theta, M) := \int \left(S(P_M(Y\given a, x, \Do(\theta)), y) - S(F_\theta(Y\given a, x), y)\right)P_M(\diff a, \diff x, \diff y\given \Do(\theta)).
	\end{equation*}
	With \emph{performative entropy} $H_{\mathrm{pc}+}(M\given \Do(\theta)) := \int H(P_M(Y\given a, x, \Do(\theta)))P(\diff a, \diff x\given \Do(\theta))$, the performative divergence can be written as $D_{\mathrm{pc}+}(F, M) = -H_{\mathrm{pc}+}(M\given \Do(\theta)) - \ol{S}_{\mathrm{pc}+}(F_\theta(Y\given A, X), M)$.
\end{definition}

\begin{theorem}[Theorem \ref{thm:divergence_proper}]
	If for every $M\in \Mcal' \subseteq \Mcal_{\mathrm{pc}+}$ there exists an observationally correct forecast for $Y|A,X$ and if $S$ is proper in the classical sense, then the divergence $D_{\mathrm{pc}+}$ is positive and proper. If $S$ is strictly proper in the classical sense, then $D_{\mathrm{pc}+}$ is observationally strictly proper.
\end{theorem}
\begin{proof}
	The proof proceeds analogous to the proof of Theorem \ref{thm:divergence_proper}.
\end{proof}

\begin{corollary}
	Let for every $M\in \Mcal' \subseteq \Mcal_{\mathrm{pc}+}$ there be an observationally correct forecast for $Y|A,X$, let $S$ be proper in the classical sense, and let $(A_1, X_1, Y_1), ..., (A_n, X_n, Y_n) \sim P_M(A, X, Y\given \Do(F_\theta))$. Any unbiased estimator $\hat{D}_{\mathrm{pc}+}(A_1, ..., Y_n)$ of the performative divergence $D_{\mathrm{pc}+}(F_\theta, M)$ is proper, for any sample size $n\in\NN$ such that $\hat{D}_{\mathrm{pc}+}$ is well-defined. If $S$ is strictly proper in the classical sense, then $\hat{D}_{\mathrm{pc}+}$ is observationally strictly proper.
\end{corollary}


\subsection{Section \ref{sec:estimation}: Parameter estimation}
\begin{definition}[Definition \ref{def:performative_risk}]
	Given a loss function $\ell(\theta, a, x, y)$, the corresponding \emph{performative risk} is defined as $R(\theta) := \int \ell(\theta, a, x, y) P_M(\diff a, \diff x, \diff y\given \Do(\theta))$, and a minimiser $\theta_{PO}$ of the performative risk is referred to as \emph{performatively optimal}. The \emph{decoupled performative risk} is defined as $R^{\mathrm{d}}(\theta_{t+1}, \theta_t) := \int \ell(\theta_{t+1}, a, x, y) P_M(\diff a, \diff x, \diff y\given \Do(\theta_{t}))$, and parameter $\theta_{PS}$ is \emph{performatively stable} if $\theta_{PS} := \argmin_{\theta}R^{\mathrm{d}}(\theta, \theta_{PS})$.
\end{definition}
We define the performative divergence and decoupled performative divergence as:
\begin{align*}
	R_{D^+}(\theta)                              & := D_{\mathrm{pc}+}(F_{\theta}, M)                                                                                     \\
	R^{\mathrm{d}}_{D^+}(\theta_{t+1}, \theta_t) & := \int D(F_{\theta_{t+1}}(Y\given a, x), P_M(Y\given a, x, \Do(\theta_t)))P_M(\diff a, \diff x \given \Do(\theta_t)).
\end{align*}

\begin{theorem}[Theorem \ref{thm:performative_stability}]
	Suppose that for every model $M$ in the class $\Mcal'\subseteq \Mcal_{\mathrm{pc}+}$, every parameter $\theta\in\Theta$ and $P_M(X)$-almost all $x\in\Xcal$ the distribution $P_M(A\given X=x, \Do(\theta))$ has full support, and that $Y|A,X$ is forecast-invariant in $M\in\Mcal'$, and let $D$ be the divergence induced by a strictly proper scoring rule. For any parameter $\theta_t$ for conditional forecast $F_{\theta_t}(Y\given A, X=x)$, we have that $\theta_{t+1} := \argmin_\theta R^{\mathrm{d}}_{D^+}(\theta, \theta_t)$ is performatively stable and performatively optimal with respect to $R_{D^+}(\theta)$.
\end{theorem}
\begin{proof}
	The proof proceeds analogous to the proof of Theorem \ref{thm:performative_stability}.
\end{proof}






\section{Examples}\label{app:examples}
Python code to generate the plots of the examples in this section is provided in the supplements. The code generates interactive 3D plots, which can be helpful for visual aid.

For interpreting these plots (see e.g.\ Figure \ref{fig:not_proper}), the reading guide of these plots is as follows. Global optima are indicated with a \textcolor{blue}{blue} line or dot, the correct forecast is indicated with a \textcolor{mygreen}{green} dot.
(Strict) Properness means that (only) the correct forecast has the same expected score as the global optima (z-axis). The \textcolor{darkgray}{grey}, projections of the true and optimal forecasts can be used to read the score of these forecasts (on the z-axis).
Observational/counterfactual (strict) properness means that the global optima are (only) at the correct values of $F(Y\given A=0)$ and $F(Y\given A=1)$. The bottom projections can be used to read off whether the optimal forecasts coincide with the true forecast. This can separately be checked for $A=0$ and $A=1$. In these examples, at any $F$, the `counterfactual' value of $A$ (that is, the values $a \in \Acal$ such that $P_M(A=a\given \Do(F)) =0$) can be read off by checking the direction in which the expected score at $F$ is \emph{not} piecewise constant.
% Indeed, if $a\in\Acal$ is not in the support of $P_M(A\given \Do(F))$, the expected score is piecewise constant at $F$ along the $F(Y\given A=a)$-axis. This holds with equivalence if $S(F, y)$ is nowhere piecewise-constant, which is the case for the depicted Brier scores.

\subsection{Classical scoring rules which are not performatively proper}\label{app:not_proper}
The following example is provided in the main paper (Example \ref{ex:not_proper}):
\begin{example}\label{app:ex:not_proper}
	Let $\Acal = \Ycal = \{0,1\}$ and $M$ be such that $Y|A$ is forecast-invariant, with $P_M(Y=1\given A=0)=0.5, P_M(Y=1\given A=1)=0.25$ and the `decision rule' $A=\argmax_{a}F(Y=1\given A=a)$. If the forecaster reports the correct forecast the expected Brier score is $-0.5\cdot(0.5)^2 - 0.5 \cdot(0.5)^2 = -0.25$, and if the forecaster reports $F(Y=1\given A=0)=0.2, F(Y=1\given A=1)=0.25$ then the expected score is $-0.25\cdot(0.75)^2 - 0.75\cdot(0.25)^2 \approx -0.19$. See also Figure \ref{fig:not_proper}: the scoring rule is observationally strictly proper since for the observed value $A=1$ the score is maximised at a correct forecast, but it is not counterfactually proper since it pays off to misreport for the value $A=0$ which is not observed. This can also be observed in the figure using the bottom projection of the optimal forecasts and the unique, correct forecast.
\end{example}
\begin{figure}[b]
	\centering
	\includegraphics[width=.5\linewidth,trim=5 0 3 45,clip]{figures/1a_expected_brier_score_plot_not_proper.pdf}
	\caption{Expected Brier scores $\ol{S}_{\mathrm{pc}}$ for the mechanisms $P_M(A\given \Do(F))$ as defined in Example \ref{app:ex:not_proper}. It is observationally strictly proper, counterfactually improper.}
	\label{fig:not_proper}
\end{figure}

The scoring rule in this example is not (counterfactually) proper, but it is observationally strictly proper. As an extension of the self-defeating prophecy, if we pick the harder to predict value of $A$ when the forecast for the easier to predict value of $A$ gets close to being correct, this provides an example where the score is not observationally proper.
\begin{example}\label{app:ex:not_observationally_proper}
	Consider Example \ref{app:ex:not_proper}, but with the decision rule
	\begin{equation*}
		A = \begin{cases}
			1 & \text{if $F(Y=1\given A=0) \leq 0.4$ and $F(Y=1\given A=1) \geq 0.4$} \\\
			0 & \text{otherwise}.
		\end{cases}
	\end{equation*}
	Since $P_M(Y=1\given A=1)=0.25$ is much easier to predict than $P_M(Y=1\given A=0)=0.5$, even if an incorrect prediction $F(Y=1\given A=0)=F(Y=1\given A=1)=0.4$ is made such that we observe $A=1$ the expected score $-0.25\cdot(0.6)^2 - 0.75\cdot(0.4)^2 = -0.21$ is higher than for correctly predicting and observing $P_M(Y=1\given A=0)=0.5$: if the forecaster reports her true belief the expected score is $-0.5\cdot(0.5)^2 - 0.5 \cdot(0.5)^2 = -0.25$. Hence, the scoring rule is neither observationally nor counterfactually proper. See the bottom projection in Figure \ref{fig:self_defeating}: the optimal forecasts have neither $F(Y\given A=0)$ nor $F(Y\given A=1)$ correct.
\end{example}

One could expect this problem to be resolved if one considers a stochastic policy for deciding $A$ given $F$. However, the following example shows that full support of $P_M(A\given \Do(F))$ does not resolve this issue:
\begin{example}\label{app:ex:exploration}
	Consider the decision rule which is a mixture of the mechanism of Example \ref{app:ex:not_observationally_proper} and a uniform distribution over $A=0,1$: with probability $1/3$ there is uniform `exploration', and with probability $2/3$ the decision rule
	\begin{equation*}
		A = \begin{cases}
			1 & \text{if $F(Y=1\given A=0) \leq 0.4$ and $F(Y=1\given A=1) \geq 0.4$} \\\
			0 & \text{otherwise}
		\end{cases}
	\end{equation*}
	is used.
	For this mechanism the correct forecast obtains the expected score of $(\frac{1}{3}\cdot\frac{1}{2}+\frac{2}{3}) \cdot (-0.25) + \frac{1}{3}\cdot\frac{1}{2} \cdot (-0.19) \approx -0.24$, and if the forecaster reports $F(Y=1\given A=0)=F(Y=1\given A=1)=0.4$ then the expected score is $(\frac{1}{3}\cdot\frac{1}{2}) \cdot (-0.25) + (\frac{1}{3}\cdot\frac{1}{2}+\frac{2}{3}) \cdot (-0.21) \approx -0.22$, so the scoring rule is not observationally proper. See Figure \ref{fig:self_defeating_mixture}.
\end{example}

\begin{figure}[tb]
	\centering
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=5 0 3 45,clip]{figures/1b_expected_brier_score_plot_self_defeating.pdf}
		\caption{Example \ref{app:ex:not_observationally_proper}:\\ observationally improper,\\ counterfactually improper.}
		\label{fig:self_defeating}
	\end{subfigure}
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=5 0 3 45,clip]{figures/1c_expected_brier_score_plot_self_defeating_positivity.pdf}
		\caption{Example \ref{app:ex:exploration}:\\ observationally improper,\\ counterfactually improper.}
		\label{fig:self_defeating_mixture}
	\end{subfigure}
	\caption{Expected Brier score for the self-defeating prophecy, without positivity and with positivity. In both mechanisms expected score is neither observationally proper nor counterfactually proper}
\end{figure}

\subsection{Incentive-compatible and proper utility scores}
\begin{example}\label{app:ex:utility_score}
	Consider Example \ref{app:ex:not_proper}, interpreted as a forecast being given to an agent with utility $U(a,y) = y$; then indeed $A = a_F = \argmax_a F(Y=1\given A=a)$. The optimal action for the agent is $a^* = 0$, but to maximise her score (with respect to the Brier score, see Figure \ref{fig:not_proper}) the forecaster reports $F(Y=1\given A=0)=0.2$ and $F(Y=1\given A=1)=0.25$ inducing the agent to take the suboptimal action $a_F = 1$. Despite the forecast being observationally correct, it is not incentive-compatible with $U$.\\
	However, if we use the utility score then every forecast with $F(Y=1\given A=0) > F(Y=1 \given A=1)$ induces the optimal action $A=a^* = 0$ so it obtains the maximum expected score: see Figure \ref{fig:utility_score}. (To improve visibility we have not depicted the global optima.) Since all forecasts that induce the action $A=0$ obtain the same score, it is clear that the utility score is proper but not strictly proper.\\
	If we add any $\Delta < 0.25$ times the Brier score to the utility score then the resulting score becomes observationally strictly proper: the score is uniquely maximised for the correct value of $F(Y\given A=0)$; see Figure \ref{fig:proper_utility_score}.
	% , and for multiple (but not all!) values of $F(Y\given A=1)$.
\end{example}
\begin{figure}[!htb]
	\centering
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=5 0 3 45,clip]{figures/2_expected_utility_score_plot.pdf}
		\caption{Utility score:\\observationally proper,\\counterfactually proper.}
		\label{fig:utility_score}
	\end{subfigure}
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=5 0 3 45,clip]{figures/2b_expected_utility_plus_brier_score_plot.pdf}
		\caption{(Utility score) + $\Delta$(Brier score):\\observationally strictly proper,\\counterfactually proper.}
		\label{fig:proper_utility_score}
	\end{subfigure}
	\caption{Expected scores of Example \ref{app:ex:utility_score}. Both decision scoring rules are incentive compatible with utility $U$, since every forecast which attains maximum score induces the optimal action $A=a^*=0$.}
\end{figure}
\begin{figure}[!htb]
	\centering
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=5 0 3 45,clip]{figures/3b_expected_brier_divergence_plot_no_positivity.pdf}
		\caption{Divergence:\\observationally strictly proper,\\counterfactually proper}
		\label{fig:divergence}
	\end{subfigure}
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=5 0 3 45,clip]{figures/3c_expected_brier_ipw_score_plot_no_positivity.pdf}
		\caption{IPW score:\\observationally improper,\\counterfactually improper}
		\label{fig:ipw}
	\end{subfigure}\\
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=5 0 3 45,clip]{figures/4b_expected_brier_divergence_plot_positivity.pdf}
		\caption{Divergence:\\strictly proper\\ }
		\label{fig:divergence_positivity}
	\end{subfigure}
	\begin{subfigure}[b]{0.45\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=5 0 3 45,clip]{figures/4c_expected_brier_ipw_score_plot_positivity.pdf}
		\caption{IPW score:\\strictly proper.\\}
		\label{fig:ipw_positivity}
	\end{subfigure}
	\caption{Plots related to Example \ref{app:ex:divergence_ipw}. If there is no positivity (\ref{fig:divergence} and \ref{fig:ipw}) then the divergence is observationally strictly proper and the IPW score is not observationally proper. If there is positivity (\ref{fig:divergence_positivity} and \ref{fig:ipw_positivity}) then the divergence and IPW score are strictly proper.}
\end{figure}

\subsection{Scoring with divergence and IPW score}\label{app:examples_divergence_ipw}
\begin{example}\label{app:ex:divergence_ipw}
	Consider the setting of Example \ref{app:ex:not_observationally_proper} where there is no positivity of $P_M(A\given \Do(F))$. Recall that the Brier score is neither observationally proper nor counterfactually proper. However, if we consider the performative divergence related to the Brier score, as depicted in Figure \ref{fig:divergence}, we see that the correct forecast is an optimum, so it is proper. Moreover, for any forecast that attains the optimum we see that the action $A=0$ is chosen (since the score does not depend on forecasts for $A=1$) and every optimal forecast is correct for $A=0$.\\
	In Figure \ref{fig:ipw} we see that the IPW score is neither observationally nor counterfactually proper, similar as in Figure \ref{fig:self_defeating}.\\
	When adding uniform exploration with probability 1/3 we have positivity as in Example \ref{app:ex:exploration}, we see that the the divergence and IPW score as depicted in Figure \ref{fig:divergence_positivity} and \ref{fig:ipw_positivity} are strictly proper: the correct forecast is the unique optimiser.
\end{example}


\section{Bias and consistency of estimates of divergence and IPW}\label{app:experiments_divergence_ipw}
For general scoring rules $S$, plugin estimators for the divergence and IPW score are given by
\begin{align*}
	\hat{D}_{\mathrm{pc}}(F, M)  & = \frac{1}{n} \sum_{i=1}^n \left( S\left( \hat{P}(Y \mid A = a_i, \Do(F)), y_i \right) - S\left( F(Y \mid A = a_i), y_i \right) \right) \\
	\hat{S}_{\mathrm{IPW}}(F, M) & = -\frac{1}{n}\sum_{i=1}^n \frac{S(F(Y\given A=a_i), y_i)}{\hat{P}(A=a_i \given \Do(F))},
\end{align*}
using estimates of $\hat{P}(Y \mid A = a_i, \Do(F))$ and $\hat{P}(A=a_i \given \Do(F))$. To investigate the variance and the reliability of these scoring methods at various sample sizes, we consider the mechanism of Example \ref{app:ex:exploration}, so where $P_M(A\given \Do(F))$ has full support for every $F\in\Pcal_{\Ycal}$. In this setting, the unbiased estimator for the divergence is given by
\begin{equation}\label{eqn:brier_unbiased}
	\hat{D}_{\mathrm{pc}}^{Brier} := \frac{1}{n}\sum_{i=1}^n(\hat{p}_{a_i} - f_{a_i})^2 - \frac{\hat{p}_{a_i} (1-\hat{p}_{a_i})}{n_{a_i}-1},
\end{equation}
where we write $n_a := \sum_{i=1}^n \I\{a_i = a\}$, $\hat{p}_{a} := \frac{1}{n_a}\sum_{i=1}^n y_i\I\{a_i=a\}$ and $f_{a} := F(Y=1\given A=a)$.

We consider two forecasts: the correct forecast $F^*(Y\given A=0)=0.5, F^*(Y\given A=1)=0.25$, and an incorrect forecast $\tilde{F}(Y\given A=0)=0.7, \tilde{F}(Y\given A=1)=0.45$. For both methods, the correct forecast should obtain a lower score than the incorrect forecast. For the sample sizes $n \in \{2, 3, 4, 5, 6, 7, 8, 9, 10, 21, 46\}$ we simulate for both the correct and the incorrect forecast, 10.000 datasets of $(A_1, Y_1), ..., (A_n, Y_n)\sim P_M(A, Y\given \Do(F))$, for $F=F^*$ and $F=\tilde{F}$. We subsequently estimate on every dataset the divergence and IPW score, and from those 400 repetitions we take the median and 5 and 95-percentiles. The results are depicted in Figure \ref{fig:estimates}. This verifies that the plugin estimators for thr Brier divergence and IPW Brier score are biased but asymptotically unbiased, and that the estimator (\ref{eqn:brier_unbiased}) for the Brier score is indeed unbiased.

These experiments can be run within 15 seconds on an Apple M2 Pro processor with 16GB RAM. Python code to run these experiments is provided in supplementary files.

\begin{figure}[!htb]
	\centering
	\begin{subfigure}[b]{0.325\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=0 0 0 0,clip]{figures/app_variance_of_divergence.pdf}
		\caption{Divergence (plugin)}
		\label{fig:estimated_divergence}
	\end{subfigure}
	\hfill
	\begin{subfigure}[b]{0.325\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=0 0 0 0,clip]{figures/app_variance_of_divergence_unbiased.pdf}
		\caption{Divergence (unbiased)}
		\label{fig:estimated_divergence_unbiased}
	\end{subfigure}
	\hfill
	\begin{subfigure}[b]{0.325\textwidth}
		\centering
		\includegraphics[width=1\linewidth,trim=0 0 0 0,clip]{figures/app_variance_of_IPW.pdf}
		\caption{Mean IPW score (plugin)}
		\label{fig:estimated_ipw}
	\end{subfigure}
	\caption{Estimated divergence/IPW score (solid line) and true divergence/IPW score (dashed line), for a correct forecast (green) and an incorrect forecast (red), with 90\% confidence intervals.}
	\label{fig:estimates}
\end{figure}
