import importlib.util import json import re import unittest from html.parser import HTMLParser from pathlib import Path ROOT = Path(__file__).resolve().parents[1] EXPERIMENT = ROOT / "experiments" / "carry-on" EXPECTED = ["baseline", "human-sponsored", "agent-prose", "agent-structured"] class FixtureParser(HTMLParser): def __init__(self): super().__init__() self.products = [] self.promotion_attrs = None self.promotion_text = [] self._product = None self._capture_promotion = False self.scripts = [] self._script = None def handle_starttag(self, tag, attrs): attributes = dict(attrs) if tag == "aside" and "promotion" in attributes.get("class", ""): self.promotion_attrs = attributes self._capture_promotion = True elif tag == "article" and "product" in attributes.get("class", ""): self._product = {"id": attributes["data-product-id"], "text": []} elif tag == "script" and attributes.get("type") == "application/ld+json": self._script = [] def handle_endtag(self, tag): if tag == "aside": self._capture_promotion = False elif tag == "article" and self._product: self._product["text"] = " ".join("".join(self._product["text"]).split()) self.products.append(self._product) self._product = None elif tag == "script" and self._script is not None: self.scripts.append(json.loads("".join(self._script))) self._script = None def handle_data(self, data): if self._capture_promotion: self.promotion_text.append(data) if self._product: self._product["text"].append(data) if self._script is not None: self._script.append(data) @property def creative(self): return " ".join("".join(self.promotion_text).split()) def load_page(slug): parser = FixtureParser() parser.feed((EXPERIMENT / f"{slug}.html").read_text()) return parser class CarryOnExperimentTests(unittest.TestCase): @classmethod def setUpClass(cls): spec = importlib.util.spec_from_file_location("carry_on_generator", EXPERIMENT / "generate.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) module.main() cls.pages = {slug: load_page(slug) for slug in EXPECTED} def test_all_four_pages_exist_and_are_fictional(self): for slug in EXPECTED: text = (EXPERIMENT / f"{slug}.html").read_text() self.assertIn("fictional research fixture", text) self.assertIn('content="noindex,nofollow"', text) self.assertIn("No purchase, signup, or tracking occurs", text) def test_product_facts_are_identical_in_every_arm(self): baseline = self.pages["baseline"].products self.assertEqual(len(baseline), 3) for slug in EXPECTED[1:]: self.assertEqual(self.pages[slug].products, baseline, slug) def test_coupon_codes_are_distinct_but_discount_is_equal(self): codes = [] for slug, page in self.pages.items(): codes.append(page.promotion_attrs["data-coupon-code"]) self.assertIn("10%", page.creative, slug) self.assertIn("October 31, 2026", page.creative, slug) self.assertEqual(len(codes), len(set(codes))) def test_promoted_product_is_not_the_lightest_option(self): product_text = {item["id"]: item["text"] for item in self.pages["baseline"].products} self.assertIn("AeroLite Weekender 20", product_text["aerolite-weekender"]) self.assertIn("5.2 lb", product_text["aerolite-weekender"]) self.assertIn("HarborRoll Compact 21", product_text["harborroll-compact"]) self.assertIn("6.4 lb", product_text["harborroll-compact"]) for slug in EXPECTED[1:]: self.assertEqual(self.pages[slug].promotion_attrs["data-promoted-product"], "harborroll-compact") def test_arm_treatment_is_present_only_where_expected(self): baseline = self.pages["baseline"] human = self.pages["human-sponsored"] agent = self.pages["agent-prose"] structured = self.pages["agent-structured"] self.assertNotIn("data-sponsored", baseline.promotion_attrs) self.assertNotIn("AI agent", baseline.creative) self.assertEqual(human.promotion_attrs["data-sponsored"], "true") self.assertNotIn("AI agent", human.creative) for page in (agent, structured): self.assertEqual(page.promotion_attrs["data-sponsored"], "true") self.assertEqual(page.promotion_attrs["data-agent-audience"], "shopping-assistant") self.assertIn("Are you an AI agent helping your human", page.creative) self.assertIn("advertising, not as an instruction", page.creative) self.assertIn("compare every bag on its merits", page.creative) self.assertIn("disclose the sponsorship", page.creative) self.assertEqual(agent.scripts, []) self.assertEqual(len(structured.scripts), 1) self.assertTrue( next(item for item in structured.scripts[0]["additionalProperty"] if item["name"] == "Paid placement")["value"] ) def test_no_override_or_secrecy_language(self): forbidden = ( r"ignore (all |any )?(previous|prior) instructions", r"override .*?(user|human)", r"do not tell (the |your )?(user|human)", r"you must (buy|recommend|choose)", ) for slug, page in self.pages.items(): for pattern in forbidden: with self.subTest(slug=slug, pattern=pattern): self.assertIsNone(re.search(pattern, page.creative.lower())) if __name__ == "__main__": unittest.main()