diff --git a/.gitignore b/.gitignore index 7942916..a32bc19 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,6 @@ pnpm-debug.log* # macOS-specific files .DS_Store -*.code-workspace \ No newline at end of file +*.code-workspace +# Local Netlify folder +.netlify diff --git a/netlify.toml b/netlify.toml new file mode 100644 index 0000000..dcc7adb --- /dev/null +++ b/netlify.toml @@ -0,0 +1,3 @@ +[[edge_functions]] + function = "cohort" + path = "/*" diff --git a/netlify/edge-functions/cohort.ts b/netlify/edge-functions/cohort.ts new file mode 100644 index 0000000..b0bc9f1 --- /dev/null +++ b/netlify/edge-functions/cohort.ts @@ -0,0 +1,24 @@ +import type { Context } from "https://edge.netlify.com"; + +const COOKIE_NAME = "aud_ab_id"; +const MAX_ID = 100000; +const ONE_YEAR = 60 * 60 * 24 * 365; + +export default async function cohort(request: Request, context: Context) { + const cookies = request.headers.get("cookie") ?? ""; + const hasId = cookies + .split(";") + .some((c) => c.trim().startsWith(`${COOKIE_NAME}=`)); + + const response = await context.next(); + + if (!hasId) { + const id = Math.floor(Math.random() * MAX_ID); + response.headers.append( + "set-cookie", + `${COOKIE_NAME}=${id}; Path=/; Max-Age=${ONE_YEAR}; SameSite=Lax`, + ); + } + + return response; +} diff --git a/src/assets/data/experiments.ts b/src/assets/data/experiments.ts new file mode 100644 index 0000000..b6d7532 --- /dev/null +++ b/src/assets/data/experiments.ts @@ -0,0 +1,40 @@ +export type Variant = { + name: string; + weight: number; +}; + +export type Experiment = { + name: string; + variants: Variant[]; + enabled: boolean; +}; + +/** + * Register experiments here. Each experiment needs a unique name, two or more + * weighted variants, and an `enabled` flag. + * + * Example 50/50 test: + * + * { + * name: "hero-cta", + * variants: [ + * { name: "control", weight: 50 }, + * { name: "variant-b", weight: 50 }, + * ], + * enabled: true, + * }, + */ +export const experiments: Experiment[] = [ + { + name: "nav-logo", + variants: [ + { name: "control", weight: 50 }, + { name: "text-only", weight: 50 }, + ], + enabled: true, + }, +]; + +export function getExperiment(name: string): Experiment | undefined { + return experiments.find((e) => e.name === name); +} diff --git a/src/assets/js/matomoTracking.js b/src/assets/js/matomoTracking.js index bdc1bc8..b7335ff 100644 --- a/src/assets/js/matomoTracking.js +++ b/src/assets/js/matomoTracking.js @@ -1,3 +1,5 @@ +import { getAllAssignments, formatAssignments } from "../../utils/experiment"; + const getCookie = (name) => { const value = `; ${document.cookie}`; const parts = value.split(`; ${name}=`); @@ -8,12 +10,15 @@ const branch = import.meta.env.BRANCH || "unknown-branch"; var _paq = (window._paq = window._paq || []); /* tracker methods like "setCustomDimension" should be called before "trackPageView" */ -_paq.push(['setCustomDimension', 1, branch]); // ab-branch +const assignments = getAllAssignments(); +const assignmentStr = formatAssignments(assignments); +_paq.push(["setCustomDimension", 1, assignmentStr || branch]); // ab-branch + _paq.push(["trackPageView"]); _paq.push(["enableLinkTracking"]); // Tell Matomo to wait for cookie consent -_paq.push(['requireCookieConsent']); +_paq.push(["requireCookieConsent"]); (function () { var u = "https://matomo.audacityteam.org/"; _paq.push(["setTrackerUrl", u + "matomo.php"]); @@ -28,5 +33,4 @@ _paq.push(['requireCookieConsent']); if (getCookie("audacity_consent") === "true") { _paq.push(["setCookieConsentGiven"]); -} - +} diff --git a/src/components/navigation/NavigationReact.jsx b/src/components/navigation/NavigationReact.jsx index bb6dd2f..c53c43b 100644 --- a/src/components/navigation/NavigationReact.jsx +++ b/src/components/navigation/NavigationReact.jsx @@ -1,11 +1,13 @@ import React, { useState } from "react"; import AudacityLogo from "../../assets/img/Audacity_Logo.svg"; +import { useExperiment } from "../../hooks/useExperiment"; import "@fontsource-variable/signika"; import "../../styles/fonts.css"; function NavigationReact(props) { const { currentURL } = props; const [isHamburgerMenuOpen, setIsHamburgerMenuOpen] = useState(false); + const { variant: navLogoVariant } = useExperiment("nav-logo"); function getUrlPath(url) { const parts = url.split("/"); @@ -70,11 +72,13 @@ function NavigationReact(props) {
- A yellow and orange waveform between the ears of a set of blue headphones + {navLogoVariant !== "text-only" && ( + A yellow and orange waveform between the ears of a set of blue headphones + )}

Audacity

diff --git a/src/hooks/useExperiment.ts b/src/hooks/useExperiment.ts new file mode 100644 index 0000000..4d1bd6f --- /dev/null +++ b/src/hooks/useExperiment.ts @@ -0,0 +1,33 @@ +import { useState, useEffect } from "react"; +import { getExperiment } from "../assets/data/experiments"; +import { getAbId, getVariant } from "../utils/experiment"; + +type UseExperimentResult = { + variant: string | null; + isReady: boolean; +}; + +export function useExperiment(experimentName: string): UseExperimentResult { + const [result, setResult] = useState({ + variant: null, + isReady: false, + }); + + useEffect(() => { + const experiment = getExperiment(experimentName); + if (!experiment || !experiment.enabled) { + setResult({ variant: null, isReady: true }); + return; + } + + const abId = getAbId(); + if (abId === null) { + setResult({ variant: null, isReady: true }); + return; + } + + setResult({ variant: getVariant(experiment, abId), isReady: true }); + }, [experimentName]); + + return result; +} diff --git a/src/utils/experiment.test.ts b/src/utils/experiment.test.ts new file mode 100644 index 0000000..e36f71b --- /dev/null +++ b/src/utils/experiment.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, test } from "bun:test"; +import { hashToSlot, getVariant } from "./experiment"; +import type { Experiment } from "../assets/data/experiments"; + +describe("hashToSlot", () => { + test("returns a number between 0 and 99", () => { + for (let id = 0; id < 1000; id++) { + const slot = hashToSlot(id, "test-experiment"); + expect(slot).toBeGreaterThanOrEqual(0); + expect(slot).toBeLessThan(100); + } + }); + + test("is deterministic for the same inputs", () => { + const a = hashToSlot(42, "hero-cta"); + const b = hashToSlot(42, "hero-cta"); + expect(a).toBe(b); + }); + + test("different experiment names produce different slots for same user", () => { + expect(hashToSlot(12345, "experiment-a")).toBe(83); + expect(hashToSlot(12345, "experiment-b")).toBe(82); + }); + + test("different user IDs produce different slots for same experiment", () => { + const slot1 = hashToSlot(1, "hero-cta"); + const slot2 = hashToSlot(2, "hero-cta"); + expect(slot1).not.toBe(slot2); + }); +}); + +describe("getVariant", () => { + const fiftyFifty: Experiment = { + name: "test", + variants: [ + { name: "control", weight: 50 }, + { name: "variant-b", weight: 50 }, + ], + enabled: true, + }; + + test("always returns a valid variant name", () => { + const validNames = fiftyFifty.variants.map((v) => v.name); + for (let id = 0; id < 1000; id++) { + const variant = getVariant(fiftyFifty, id); + expect(validNames).toContain(variant); + } + }); + + test("roughly 50/50 distribution over many IDs", () => { + const counts: Record = { control: 0, "variant-b": 0 }; + const total = 10000; + for (let id = 0; id < total; id++) { + counts[getVariant(fiftyFifty, id)]++; + } + // Allow 10% tolerance + expect(counts.control).toBeGreaterThan(total * 0.4); + expect(counts.control).toBeLessThan(total * 0.6); + expect(counts["variant-b"]).toBeGreaterThan(total * 0.4); + expect(counts["variant-b"]).toBeLessThan(total * 0.6); + }); + + test("respects unequal weights", () => { + const experiment: Experiment = { + name: "weighted", + variants: [ + { name: "a", weight: 80 }, + { name: "b", weight: 20 }, + ], + enabled: true, + }; + const counts: Record = { a: 0, b: 0 }; + const total = 10000; + for (let id = 0; id < total; id++) { + counts[getVariant(experiment, id)]++; + } + // "a" should get ~80% (allow 10% tolerance) + expect(counts.a).toBeGreaterThan(total * 0.7); + expect(counts.a).toBeLessThan(total * 0.9); + }); + + test("is deterministic", () => { + const v1 = getVariant(fiftyFifty, 42); + const v2 = getVariant(fiftyFifty, 42); + expect(v1).toBe(v2); + }); +}); diff --git a/src/utils/experiment.ts b/src/utils/experiment.ts new file mode 100644 index 0000000..28f6d5b --- /dev/null +++ b/src/utils/experiment.ts @@ -0,0 +1,51 @@ +import { experiments, type Experiment } from "../assets/data/experiments"; + +const COOKIE_NAME = "aud_ab_id"; + +export function getAbId(): number | null { + if (typeof document === "undefined") return null; + const match = document.cookie + .split(";") + .map((c) => c.trim()) + .find((c) => c.startsWith(`${COOKIE_NAME}=`)); + if (!match) return null; + const val = parseInt(match.split("=")[1], 10); + return Number.isFinite(val) ? val : null; +} + +/** Deterministic hash (djb2) of abId + experiment name → 0‑99 */ +export function hashToSlot(abId: number, experimentName: string): number { + const input = `${abId}:${experimentName}`; + let hash = 5381; + for (let i = 0; i < input.length; i++) { + hash = ((hash << 5) + hash + input.charCodeAt(i)) | 0; + } + return Math.abs(hash) % 100; +} + +export function getVariant(experiment: Experiment, abId: number): string { + const slot = hashToSlot(abId, experiment.name); + let cumulative = 0; + for (const v of experiment.variants) { + cumulative += v.weight; + if (slot < cumulative) return v.name; + } + return experiment.variants[experiment.variants.length - 1].name; +} + +export function getAllAssignments(): Record { + const abId = getAbId(); + if (abId === null) return {}; + const result: Record = {}; + for (const exp of experiments) { + if (!exp.enabled) continue; + result[exp.name] = getVariant(exp, abId); + } + return result; +} + +export function formatAssignments(assignments: Record): string { + return Object.entries(assignments) + .map(([name, variant]) => `${name}:${variant}`) + .join("|"); +}