AB Testing.tex

\documentclass[12pt]{article}

% Packages
\usepackage[utf8]{inputenc}
\usepackage{amsmath, amsfonts, amssymb}
\usepackage{graphicx}
\usepackage{booktabs}
\usepackage{listings}
\usepackage{xcolor}
\usepackage{geometry}
\geometry{margin=1in}

% Code styling
\lstset{
    basicstyle=\ttfamily\footnotesize,
    backgroundcolor=\color{gray!10},
    frame=single,
    breaklines=true
}

\title{A/B Test Project: Email Subject Line Experiment}
\author{Analyst Report}
\date{\today}

\begin{document}

\maketitle

\section{Background}
We want to test whether a new email subject line (Variant B) improves performance compared to the existing subject line (Variant A). A/B testing provides an evidence-based way to determine if the difference is meaningful or just due to chance.

\section{Objective}
\begin{itemize}
    \item \textbf{Primary Metric:} Conversion rate
    \item \textbf{Secondary Metrics:} Open rate, Click-Through Rate (CTR)
\end{itemize}

\section{Experiment Design}
\begin{itemize}
    \item Population: 1,200 email recipients
    \item Random Split: 50\% to Variant A, 50\% to Variant B
    \item Data Collected:
    \begin{itemize}
        \item user\_id
        \item send\_time
        \item variant (A/B)
        \item opened (1 = yes, 0 = no)
        \item clicked (1 = yes, 0 = no)
        \item converted (1 = yes, 0 = no)
    \end{itemize}
\end{itemize}

\section{Results}

\subsection{Funnel Metrics}
\begin{center}
\begin{tabular}{lccc}
\toprule
Variant & Open Rate & CTR & Conversion Rate \\
\midrule
A (Control) & 38\% & 12\% & 8.0\% \\
B (Variant) & 42\% & 14\% & 11.5\% \\
\bottomrule
\end{tabular}
\end{center}

\subsection{Statistical Test (Conversion Rate)}
\begin{itemize}
    \item Variant A: 48/600 = 8.0\%
    \item Variant B: 69/600 = 11.5\%
    \item Absolute Difference: +3.5 percentage points
    \item Relative Lift: +43.75\%
    \item Z = 2.30, p-value = 0.021
    \item 95\% CI (B $-$ A): [0.5\%, 6.5\%]
\end{itemize}

\section{Python Code (Analysis)}
\subsection{Load and Summarize Data}
\begin{lstlisting}[language=Python]
import pandas as pd

df = pd.read_csv("ab_test_email_campaign.csv", parse_dates=["send_time"])
summary = df.groupby("variant")[["opened","clicked","converted"]].sum()
counts = df["variant"].value_counts()
print(summary, counts)
\end{lstlisting}

\subsection{Two-Proportion Z-Test}
\begin{lstlisting}[language=Python]
import math

def normal_cdf(z):
    return 0.5 * (1 + math.erf(z / math.sqrt(2)))

def two_proportion_z_test(success_a, n_a, success_b, n_b):
    p_pool = (success_a + success_b) / (n_a + n_b)
    se = math.sqrt(p_pool * (1 - p_pool) * (1/n_a + 1/n_b))
    z = (success_b/n_b - success_a/n_a) / se
    p_value = 2 * (1 - normal_cdf(abs(z)))
    return z, p_value
\end{lstlisting}

\subsection{Confidence Interval and Results}
\begin{lstlisting}[language=Python]
def ci_diff(success_a, n_a, success_b, n_b, alpha=0.05):
    p_a = success_a / n_a
    p_b = success_b / n_b
    se = math.sqrt(p_a*(1-p_a)/n_a + p_b*(1-p_b)/n_b)
    zcrit = 1.96
    diff = p_b - p_a
    return (diff - zcrit*se, diff + zcrit*se)

# Apply
conv_a = 48; conv_b = 69
n_a = 600; n_b = 600

z, p = two_proportion_z_test(conv_a, n_a, conv_b, n_b)
ci = ci_diff(conv_a, n_a, conv_b, n_b)

print(z, p, ci)
\end{lstlisting}

\section{Insights}
\begin{itemize}
    \item Variant B achieved a higher conversion rate (11.5\%) than A (8.0\%).
    \item The uplift of +43.7\% is statistically significant (p = 0.021).
    \item 95\% CI indicates a true improvement between 0.5\% and 6.5\%.
    \item Funnel-wide improvements (opens $\rightarrow$ clicks $\rightarrow$ conversions).
\end{itemize}

\section{Recommendation}
Adopt Variant B as the new default subject line. It provides a clear, statistically valid uplift in conversions, with no evidence of negative impacts.

\section{Next Steps}
\begin{itemize}
    \item Run follow-up experiments on personalization (e.g., using recipient names).
    \item Test different send times (morning vs evening).
    \item Segment analysis (mobile vs desktop, geographic regions).
    \item Use power analysis to determine minimum sample size for future tests.
\end{itemize}

\section{Conclusion}
This A/B test shows that Variant B outperforms Variant A, with a meaningful and statistically significant increase in conversions. Decision: roll out Variant B to all users in upcoming campaigns.

\end{document}