% flaky-tests-ci-tdd.tex — flaky-test causes/fixes, random+parallel, coverage limits, CI, TDD/BDD, traps.
% Sources: docs/memos/testing-coverage-ci-flaky.md, docs/memos/testing-traps.md,
%          docs/memos/testing-tdd.md, docs/memos/testing-tdd-worked-kata.md.
% Build ONLY with: tools/print/print-sheet.py <this>.tex --dry-run
% @source: hiot monorepo, docs/school/sheets/testing/flaky-tests-ci-tdd.tex — the SOURCE OF TRUTH; a copy anywhere else (e.g. artur.gurgul.pro) is regenerated from it, never edited
% @labels: area=testing kind=process level=senior platform=general new=no round=round2-2026-09-23 topic=testing,build
% @tags: flaky-test, test-clock, random-order, parallel-testing, code-coverage, xccov, xcodebuild-test, tdd, red-green-refactor, triangulation, bdd, test-induced-design-damage
\documentclass[8pt]{extarticle}
\usepackage{printup-sheet}
\usepackage{array}

\lstdefinelanguage{SwiftSheet}{
  morekeywords={protocol,class,final,struct,enum,func,var,let,weak,init,override,
    if,else,return,guard,self,nil,try,await,async,throws,private,some,in,
    true,false,AnyObject,Void,String,Bool,Int,import,get},
  sensitive=true, morecomment=[l]{//}, morecomment=[s]{/*}{*/}, morestring=[b]"}

\tikzset{
  sb/.style={box, font=\scriptsize, inner sep=1.5pt, minimum height=4.4mm, text width=24mm},
  cause/.style={sb, draw=sheetRed, fill=sheetRed!7},
  fix/.style={font=\tiny, text=sheetGreen!55!black, align=left, text width=24mm, inner sep=1pt},
  lbl/.style={font=\tiny, text=black!75, inner sep=1pt, align=center},
  ph/.style={circle, draw, thick, minimum size=10mm, font=\scriptsize\bfseries, align=center, inner sep=0pt},
}

\begin{document}

\sheettitle{Flaky tests · CI · coverage · TDD}{testing · memo}

\oneliner{A flaky test has a \textbf{nondeterminism source} — shared state, order, real time,
real network, animations, concurrency, environment — so \textbf{remove the source, never retry}.
Coverage says what \emph{ran}, not what was \emph{verified}. TDD = \textbf{Red → Green →
Refactor} in minute-long cycles; the failing test comes first.}

\begin{multicols}{2}

\section{Picture — flaky cause → fix}
\noindent\begin{tikzpicture}[sheet]
  \node[box, draw=sheetRed, fill=sheetRed!15, font=\small\bfseries] (f) at (3.95,0) {FLAKY};
  \foreach \n/\x/\y/\c/\fx in {
     a/0/1.35/{shared mutable state}/{fresh SUT per test, nil in tearDown; in-memory store; injected UserDefaults suite},
     b/0/0/{order dependence}/{every test builds its own world; run in random order to expose it},
     c/0/-1.35/{real time: sleep, Date()}/{inject Clock / TestClock; advance time, don't wait},
     d/5.2/1.35/{real network}/{stub the raw Data/HTTP (URLProtocol) or a local fixture server},
     e/5.2/0/{animations · UI timing}/{disable animations; waitForExistence, check isHittable},
     f2/5.2/-1.35/{concurrency · un-awaited tasks}/{await the result; expectation / confirmation; no fixed timeouts}}
  {
    \node[cause, anchor=west] (\n) at (\x,\y) {\c};
    \node[fix, anchor=north west] at (\n.south west) {\fx};
  }
  \foreach \n in {a,b,c} \draw[flow, draw=sheetRed!70] (\n.east) -- (f);
  \foreach \n in {d,e,f2} \draw[flow, draw=sheetRed!70] (\n.west) -- (f);
  \node[lbl, text=sheetRed] at (3.95,-1.75) {also: \texttt{UUID()}/random\\(seed/inject),\\locale · TZ · OS\\(pin on CI)};
\end{tikzpicture}

\section{How it works}
\begin{itemize}\raggedright
  \item \textbf{Random order + parallel} (test plan, or
        \texttt{-parallel-testing-enabled YES}): Xcode spreads test \emph{classes} over
        simulator clones. A suite must pass in \emph{any} order — if randomizing flips results
        you have hidden coupling. Last resort: \texttt{@Suite(.serialized)}.
  \item \textbf{Retry is not a fix} — a flake is often a real race users also hit; quarantine
        with a ticket, then root-cause.
  \item \textbf{Coverage} = \% of lines/branches \emph{executed} (LLVM profdata). Enable:
        scheme → Test → Options → Gather coverage, or test plan; read with
        \texttt{xcrun xccov view --report R.xcresult}. \textbf{Branch} coverage beats line
        coverage (a \texttt{guard}'s else path). Use it to \emph{find untested risk}, watch the
        trend; don't gate on a \% (teams aim $\sim$70--85\% on logic modules).
  \item \textbf{CI:} \texttt{xcodebuild test -scheme App -destination
        'platform=iOS Simulator,name=iPhone 15,OS=17.5' -resultBundlePath R.xcresult
        -enableCodeCoverage YES} (or \texttt{fastlane scan}). \textbf{Pin} simulator
        \emph{and} OS; keep the \texttt{.xcresult}, screenshots, snapshot diffs as artifacts;
        fast PR plan + slow nightly plan (locales × sanitizers).
  \item \textbf{TDD three laws} (Uncle Bob): (1) no production code without a failing test;
        (2) no more test than enough to fail — not compiling counts; (3) no more production code
        than enough to pass. \textbf{Fake it} → \textbf{triangulate} (a 2nd example forces the
        general rule). \textbf{Refactor only on green}; refactor = behaviour-preserving.
  \item \textbf{BDD}: same loop, phrased as behaviour — \textbf{Given / When / Then} (=
        Arrange/Act/Assert); Quick/Nimble, Cucumber/Gherkin.
        \textbf{London} (outside-in, mockist) vs \textbf{Detroit/Chicago} (inside-out, classicist).
  \item \textbf{Test-induced design damage} (DHH vs Beck/Fowler, ``Is TDD dead?'' 2014):
        bending code \emph{only} for tests — one-impl protocols, exposed internals. Counter:
        damage comes from over-mocking; testability usually = good decoupling.
\end{itemize}

\columnbreak

\section{Example — kill a time flake}
\begin{lstlisting}[language=SwiftSheet]
// FLAKY: real clock -- 2 s slow, fails on a loaded CI box
func test_expires_bad() {
  let t = Token(ttl: 1); sleep(2); XCTAssertTrue(t.isExpired) }

// DETERMINISTIC: inject the clock, move time yourself
protocol Clock { var now: Date { get } }
final class TestClock: Clock {
  var now = Date(timeIntervalSince1970: 0) }
func test_expires_afterTTL() {
  let clock = TestClock()
  let t = Token(ttl: 60, clock: clock)
  clock.now += 59; XCTAssertFalse(t.isExpired)   // boundary
  clock.now += 2;  XCTAssertTrue(t.isExpired)    // instant
}
\end{lstlisting}

\section{Red → Green → Refactor}
\noindent\begin{tikzpicture}[sheet]
  \node[ph, draw=sheetRed, fill=sheetRed!12] (r) at (0,0) {RED};
  \node[ph, draw=sheetGreen, fill=sheetGreen!15] (g) at (2.6,0) {GREEN};
  \node[ph, draw=sheetBlue, fill=sheetBlue!10] (f) at (1.3,-1.45) {RE-\\FACTOR};
  \draw[hot, bend left=25] (r) to node[lbl, above]{least code to pass} (g);
  \draw[hot, bend left=25] (g) to node[lbl, right=1.5mm, text width=14mm]{clean up,\\tests stay green} (f);
  \draw[hot, bend left=25] (f) to node[lbl, left=1.5mm, text width=13mm, align=right]{next tiny\\failing test} (r);
  \node[note, anchor=west, text width=36mm] at (4.0,-0.55)
    {\textbf{see it fail} for the right reason — a test that never went red proves nothing.
     Kata: \texttt{1→"I"} (fake it), \texttt{2→"II"} (triangulate), \texttt{5→"V"},
     \texttt{4→"IV"} → a value table; new numerals are \emph{data}. Bug fix = failing repro first.};
\end{tikzpicture}

\section{Interview traps (the catalogue)}
\begin{itemize}
  \trap{\textbf{Tautology:} stub returns \texttt{User(id:1)}, assert \texttt{User(id:1)} —
        stub the \emph{raw \texttt{Data}} so your parsing runs.}
  \trap{Testing the framework (\texttt{JSONDecoder} works) instead of your mapping.}
  \trap{Asserting \texttt{private} state / partial-mocking the SUT → refactor-fragile.}
  \trap{Over-mocking: all green, prod broken — fakes lie. Add contract + integration tests.}
  \trap{Ordered \texttt{==} on \texttt{Set}/\texttt{Dictionary} output → sort or compare sets.}
  \trap{No assertion = inflates coverage, tests nothing. 12 asserts in one test = poor localization.}
  \trap{Test written after the fix: revert the fix — if it doesn't go red, it proves nothing.}
  \trap{``100\% coverage'' mandate → assertion-free tests, punished deletions.}
\end{itemize}

\section{Remember}
\textbf{Control time, don't wait on it.} Isolate state · stub I/O · pin env · random order.
Coverage finds gaps, not quality. \textbf{Fail first, then pass, then clean.}

\section{Likely questions}
\begin{enumerate}
  \item Top flaky causes? — shared state, order, sleep/time, network, animation, concurrency.
  \item Passes locally, fails on CI? — locale/TZ, simulator/OS, parallel coupling, speed.
  \item Is 100\% coverage good? — executed $\neq$ verified; use branch coverage on risk.
  \item Three laws of TDD? — failing test first; just enough test; just enough code.
  \item When does TDD fit badly? — exploratory UI/animation, spikes, thin glue.
\end{enumerate}

\end{multicols}

\noindent{\footnotesize\color{sheetGrey}\textit{Related:} testing fundamentals (pyramid, doubles) ·
XCTest \& Swift Testing · testing ViewModels / UI / snapshot · DI · concurrency (actors, awaiting)}

\end{document}