example data part 1 - cache html files

aocd/models.py

@@ -7,6 +7,7 @@
 import webbrowser
 from datetime import datetime
 from datetime import timedelta
+from functools import cache
 from pathlib import Path
 from textwrap import dedent
 
@@ -165,16 +166,15 @@ def __init__(self, year, day, user=None):
         self._user = user
         self.input_data_url = self.url + "/input"
         self.submit_url = self.url + "/answer"
-        fname = f"{self.year}_{self.day:02d}"
-        pre = self.user.memo_dir / fname
+        pre = self.user.memo_dir / f"{self.year}_{self.day:02d}"
         self.input_data_fname = pre.with_name(pre.name + "_input.txt")
-        self.example_input_data_fname = pre.with_name(pre.name + "_example_input.txt")
         self.answer_a_fname = pre.with_name(pre.name + "a_answer.txt")
         self.answer_b_fname = pre.with_name(pre.name + "b_answer.txt")
         self.incorrect_answers_a_fname = pre.with_name(pre.name + "a_bad_answers.txt")
         self.incorrect_answers_b_fname = pre.with_name(pre.name + "b_bad_answers.txt")
-        self.title_fname = AOCD_DATA_DIR / "titles" / f"{self.year}_{self.day:02d}.txt"
-        self._title = ""
+        self.prose0_fname = AOCD_DATA_DIR / "prose" / (pre.name + "_prose.0.html")
+        self.prose1_fname = pre.with_name(pre.name + "_prose.1.html")  # part a solved
+        self.prose2_fname = pre.with_name(pre.name + "_prose.2.html")  # part b solved
 
     @property
     def user(self):
@@ -207,30 +207,28 @@ def input_data(self):
 
     @property
     def example_data(self):
-        try:
-            data = self.example_input_data_fname.read_text()
-        except FileNotFoundError:
-            pass
-        else:
-            log.debug("reusing existing example data %s", self.example_input_data_fname)
-            return data.rstrip("\r\n")
-        soup = self._soup()
+        text = self._get_prose()
+        soup = bs4.BeautifulSoup(text, "html.parser")
         try:
             data = soup.pre.text
         except Exception:
-            log.info("unable to find example data year=%s day=%s", self.year, self.day)
+            log.warning("unable to find example data for %d/%02d", self.year, self.day)
             data = ""
-        log.info("saving the example data")
-        atomic_write_file(self.example_input_data_fname, data)
         return data.rstrip("\r\n")
 
     @property
+    @cache
     def title(self):
-        if self.title_fname.is_file():
-            self._title = self.title_fname.read_text().strip()
-        else:
-            self._save_title()
-        return self._title
+        prose = self._get_prose()
+        soup = bs4.BeautifulSoup(prose, "html.parser")
+        if soup.h2 is None:
+            raise AocdError("heading not found")
+        txt = soup.h2.text
+        prefix = f"--- Day {self.day}: "
+        suffix = " ---"
+        if not txt.startswith(prefix) or not txt.endswith(suffix):
+            raise AocdError(f"unexpected h2 text: {txt}")
+        return txt.removeprefix(prefix).removesuffix(suffix)
 
     def _repr_pretty_(self, p, cycle):
         # this is a hook for IPython's pretty-printer
@@ -439,7 +437,6 @@ def _check_already_solved(self, guess, part):
 
     def _save_correct_answer(self, value, part):
         fname = getattr(self, f"answer_{part}_fname")
-        _ensure_intermediate_dirs(fname)
         txt = value.strip()
         msg = "saving"
         if fname.is_file():
@@ -451,30 +448,20 @@ def _save_correct_answer(self, value, part):
             msg = "overwriting"
         msg += " the correct answer for %d/%02d part %s: %s"
         log.info(msg, self.year, self.day, part, txt)
+        _ensure_intermediate_dirs(fname)
         fname.write_text(txt)
 
     def _save_incorrect_answer(self, value, part, extra=""):
         fname = getattr(self, f"incorrect_answers_{part}_fname")
-        _ensure_intermediate_dirs(fname)
         msg = "appending an incorrect answer for %d/%02d part %s"
         log.info(msg, self.year, self.day, part)
-        fname.write_text(value.strip() + " " + extra.replace("\n", " ") + "\n")
-
-    def _save_title(self, soup=None):
-        if soup is None:
-            soup = self._soup()
-        if soup.h2 is None:
-            log.warning("heading not found")
-            return
-        txt = soup.h2.text.strip("- ")
-        prefix = f"Day {self.day}: "
-        if not txt.startswith(prefix):
-            log.error("weird heading, wtf? %s", txt)
-            return
-        txt = self._title = txt[len(prefix) :]
-        _ensure_intermediate_dirs(self.title_fname)
-        with self.title_fname.open("w") as f:
-            print(txt, file=f)
+        _ensure_intermediate_dirs(fname)
+        if fname.is_file():
+            txt = fname.read_text()
+        else:
+            txt = ""
+        txt += value.strip() + " " + extra.replace("\n", " ") + "\n"
+        fname.write_text(txt)
 
     def _get_answer(self, part):
         """
@@ -486,20 +473,10 @@ def _get_answer(self, part):
         answer_fname = getattr(self, f"answer_{part}_fname")
         if answer_fname.is_file():
             return answer_fname.read_text().strip()
-        # scrape puzzle page for any previously solved answers
-        soup = self._soup()
-        if not self._title:
-            # may as well save this while we're here
-            self._save_title(soup=soup)
-        hit = "Your puzzle answer was"
-        paras = [p for p in soup.find_all("p") if p.text.startswith(hit)]
-        if paras:
-            parta_correct_answer = paras[0].code.text
-            self._save_correct_answer(value=parta_correct_answer, part="a")
-            if len(paras) > 1:
-                _p1, p2 = paras
-                partb_correct_answer = p2.code.text
-                self._save_correct_answer(value=partb_correct_answer, part="b")
+        # check puzzle page for any previously solved answers.
+        # if these were solved by typing into the website directly, rather than using
+        # aocd submit, then our caches might not know about the answers yet.
+        self._request_puzzle_page()
         if answer_fname.is_file():
             return answer_fname.read_text().strip()
         msg = f"Answer {self.year}-{self.day}{part} is not available"
@@ -546,17 +523,58 @@ def my_stats(self):
         result = stats[self.year, self.day]
         return result
 
-    def _soup(self):
-        response = http.request("GET", self.url, headers=http.headers | self.user.auth)
-        if response.status >= 400:
+    def _request_puzzle_page(self):
+        headers = http.headers | self.user.auth
+        response = http.request("GET", self.url, headers=headers)
+        if response.status != 200:
+            log.error("got %s status code", response.status)
+            log.error(response.data.decode(errors="replace"))
             raise AocdError(f"HTTP {response.status} at {self.url}")
         self._last_resp = response
-        soup = bs4.BeautifulSoup(response.data, "html.parser")
-        return soup
+        text = response.data.decode()
+        soup = bs4.BeautifulSoup(text, "html.parser")
+        hit = "Your puzzle answer was"
+        if "Both parts of this puzzle are complete!" in text:  # solved
+            if not self.prose2_fname.is_file():
+                self.prose2_fname.write_text(text)
+            hits = [p for p in soup.find_all("p") if p.text.startswith(hit)]
+            if self.day == 25:
+                [pa] = hits
+            else:
+                pa, pb = hits
+                self._save_correct_answer(pb.code.text, "b")
+            self._save_correct_answer(pa.code.text, "a")
+        elif "The first half of this puzzle is complete!" in text:  # part b unlocked
+            if not self.prose1_fname.is_file():
+                self.prose1_fname.write_text(text)
+            [pa] = [p for p in soup.find_all("p") if p.text.startswith(hit)]
+            self._save_correct_answer(pa.code.text, "a")
+        else:  # init, or dead token - doesn't really matter
+            if not self.prose0_fname.is_file():
+                _ensure_intermediate_dirs(self.prose0_fname)
+                self.prose0_fname.write_text(text)
+
+    def _get_prose(self):
+        # prefer to return full prose (i.e. part b is solved or unlocked)
+        # prefer to return prose with answers from same the user id as self.user.id
+        for path in self.prose2_fname, self.prose1_fname:
+            if path.is_file():
+                return path.read_text()
+            # see if other user has cached it
+            other = next(AOCD_DATA_DIR.glob("*/" + path.name), None)
+            if other is not None:
+                return other.read_text()
+        if self.prose0_fname.is_file():
+            return self.prose0_fname.read_text()
+        self._request_puzzle_page()
+        for path in self.prose2_fname, self.prose1_fname, self.prose0_fname:
+            if path.is_file():
+                return path.read_text()
 
     @property
     def easter_eggs(self):
-        soup = self._soup()
+        txt = self._get_prose()
+        soup = bs4.BeautifulSoup(txt, "html.parser")
         # Most puzzles have exactly one easter-egg, but 2018/12/17 had two..
         eggs = soup.find_all(["span", "em", "code"], class_=None, attrs={"title": bool})
         return eggs

aocd/post.py

@@ -23,7 +23,8 @@ def submit(
         year = most_recent_year()
     puzzle = Puzzle(year=year, day=day, user=user)
     if part is None:
-        # guess if user is submitting for part a or part b
+        # guess if user is submitting for part a or part b,
+        # based on whether part a is already solved or not
         answer_a = getattr(puzzle, "answer_a", None)
         log.warning("answer a: %s", answer_a)
         if answer_a is None:

tests/test_models.py

@@ -59,6 +59,8 @@ def test_answered(aocd_data_dir):
     assert puzzle.answered("a") is False
     assert puzzle.answered_b is True
     assert puzzle.answered("b") is True
+    with pytest.raises(AocdError('part must be "a" or "b"')):
+        puzzle.answered(1)
 
 
 def test_setattr_submits(mocker, pook):
@@ -148,24 +150,34 @@ def test_solve_for_unfound_user(aocd_data_dir, mocker):
     other_plug.load.return_value.assert_not_called()
 
 
+def test_get_title_failure_no_heading(freezer, pook, caplog):
+    freezer.move_to("2018-12-01 12:00:00Z")
+    pook.get(
+        url="https://adventofcode.com/2018/day/1",
+        response_body="--- Day 1: hello ---",
+    )
+    puzzle = Puzzle(year=2018, day=1)
+    with pytest.raises(AocdError("heading not found")):
+        puzzle.title
+
+
 def test_get_title_failure(freezer, pook, caplog):
     freezer.move_to("2018-12-01 12:00:00Z")
     pook.get(
         url="https://adventofcode.com/2018/day/1",
-        response_body="<h2>Day 11: This SHOULD be day 1</h2>",
+        response_body="<h2>--- Day 11: This SHOULD be day 1 ---</h2>",
     )
     puzzle = Puzzle(year=2018, day=1)
-    assert not puzzle.title
-    msg = "weird heading, wtf? Day 11: This SHOULD be day 1"
-    log_event = ("aocd.models", logging.ERROR, msg)
-    assert log_event in caplog.record_tuples
+    msg = "unexpected h2 text: --- Day 11: This SHOULD be day 1 ---"
+    with pytest.raises(AocdError(msg)):
+        puzzle.title
 
 
 def test_pprint(freezer, pook, mocker):
     freezer.move_to("2018-12-01 12:00:00Z")
     pook.get(
         url="https://adventofcode.com/2018/day/1",
-        response_body="<h2>Day 1: The Puzzle Title</h2>",
+        response_body="<h2>--- Day 1: The Puzzle Title ---</h2>",
     )
     puzzle = Puzzle(year=2018, day=1)
     assert puzzle.title == "The Puzzle Title"
@@ -181,7 +193,7 @@ def test_pprint_cycle(freezer, pook, mocker):
     freezer.move_to("2018-12-01 12:00:00Z")
     pook.get(
         url="https://adventofcode.com/2018/day/1",
-        response_body="<h2>Day 1: The Puzzle Title</h2>",
+        response_body="<h2>--- Day 1: The Puzzle Title ---</h2>",
     )
     puzzle = Puzzle(year=2018, day=1)
     assert puzzle.title == "The Puzzle Title"
@@ -366,12 +378,10 @@ def test_example_data_cache(aocd_data_dir, pook):
         response_body="<pre><code>1\n2\n3\n</code></pre><pre><code>annotated</code></pre>",
         times=1,
     )
-    cached = aocd_data_dir / "testauth.testuser.000/2018_01_example_input.txt"
-    assert not cached.exists()
     puzzle = Puzzle(day=1, year=2018)
+    assert mock.calls == 0
     assert puzzle.example_data == "1\n2\n3"
     assert mock.calls == 1
-    assert cached.read_text() == "1\n2\n3\n"
     assert puzzle.example_data == "1\n2\n3"
     assert mock.calls == 1
 
@@ -384,6 +394,15 @@ def test_example_data_fail(pook):
         puzzle.example_data
 
 
+def test_example_data_missing(pook, caplog):
+    url = "https://adventofcode.com/2018/day/1"
+    pook.get(url, reply=200, response_body="wat")
+    puzzle = Puzzle(day=1, year=2018)
+    assert puzzle.example_data == ""
+    record = ("aocd.models", logging.WARNING, "unable to find example data for 2018/01")
+    assert record in caplog.record_tuples
+
+
 @pytest.mark.parametrize(
     "v_raw,v_expected,len_logs",
     [
@@ -406,3 +425,14 @@ def test_type_coercions(v_raw, v_expected, len_logs, caplog):
     v_actual = p._coerce_val(v_raw)
     assert v_actual == v_expected, f"{type(v_raw)} {v_raw})"
     assert len(caplog.records) == len_logs
+
+
+def test_get_prose_cache(aocd_data_dir):
+    cached = aocd_data_dir / "other-user-id" / "2022_01_prose.2.html"
+    cached.parent.mkdir()
+    cached.write_text("foo")
+    puzzle = Puzzle(year=2022, day=1)
+    assert puzzle._get_prose() == "foo"
+    my_cached = aocd_data_dir / "testauth.testuser.000" / "2022_01_prose.2.html"
+    my_cached.write_text("bar")
+    assert puzzle._get_prose() == "bar"

tests/test_runner.py

@@ -169,10 +169,10 @@ def test_day_out_of_range(mocker, capsys, freezer):
 
 
 def test_run_error(aocd_data_dir, mocker, capsys):
-    title_path = aocd_data_dir / "titles"
-    title_path.mkdir()
-    title_file = title_path / "2018_25.txt"
-    title_file.write_text("The Puzzle Title")
+    prose_dir = aocd_data_dir / "prose"
+    prose_dir.mkdir()
+    puzzle_file = prose_dir / "2018_25_prose.0.html"
+    puzzle_file.write_text("<h2>--- Day 25: The Puzzle Title ---</h2>")
     input_path = aocd_data_dir / "testauth.testuser.000" / "2018_25_input.txt"
     input_path.write_text("someinput")
     answer_path = aocd_data_dir / "testauth.testuser.000" / "2018_25a_answer.txt"
@@ -195,10 +195,10 @@ def test_run_error(aocd_data_dir, mocker, capsys):
 
 
 def test_run_and_autosubmit(aocd_data_dir, mocker, capsys, pook):
-    title_path = aocd_data_dir / "titles"
-    title_path.mkdir()
-    title_file = title_path / "2015_01.txt"
-    title_file.write_text("The Puzzle Title")
+    prose_dir = aocd_data_dir / "prose"
+    prose_dir.mkdir()
+    puzzle_file = prose_dir / "2015_01_prose.0.html"
+    puzzle_file.write_text("<h2>--- Day 1: The Puzzle Title ---</h2>")
     input_path = aocd_data_dir / "testauth.testuser.000" / "2015_01_input.txt"
     input_path.write_text("testinput")
     answer_path = aocd_data_dir / "testauth.testuser.000" / "2015_01a_answer.txt"
@@ -224,10 +224,10 @@ def test_run_and_autosubmit(aocd_data_dir, mocker, capsys, pook):
 
 
 def test_run_and_no_autosubmit(aocd_data_dir, mocker, capsys, pook):
-    title_path = aocd_data_dir / "titles"
-    title_path.mkdir()
-    title_file = title_path / "2015_01.txt"
-    title_file.write_text("The Puzzle Title")
+    prose_dir = aocd_data_dir / "prose"
+    prose_dir.mkdir()
+    puzzle_file = prose_dir / "2015_01_prose.0.html"
+    puzzle_file.write_text("<h2>--- Day 1: The Puzzle Title ---</h2>")
     input_path = aocd_data_dir / "testauth.testuser.000" / "2015_01_input.txt"
     input_path.write_text("testinput")
     answer_path = aocd_data_dir / "testauth.testuser.000" / "2015_01a_answer.txt"