fix: mark notes for redownload if they are corrupted in DB

Author: vzhd1701

evernote_backup/note_storage.py

@@ -295,26 +295,41 @@ def iter_notes(self, notebook_guid: str) -> Iterator[Note]:
         for note_guid in self._get_notes_by_notebook(notebook_guid):
             with self.db as con:
                 cur = con.execute(
-                    "select raw_note"
+                    "select title, guid, raw_note"
                     " from notes"
                     " where guid=? and raw_note is not NULL",
                     (note_guid,),
                 )
 
                 row = cur.fetchone()
 
-                yield pickle.loads(lzma.decompress(row["raw_note"]))
+                raw_note = self._get_raw_note(
+                    row["title"],
+                    row["guid"],
+                    row["raw_note"],
+                )
+
+                if raw_note:
+                    yield raw_note
 
     def iter_notes_trash(self) -> Iterator[Note]:
         with self.db as con:
             cur = con.execute(
-                "select raw_note"
+                "select title, guid, raw_note"
                 " from notes"
                 " where is_active=0 and raw_note is not NULL"
                 " order by title COLLATE NOCASE",
             )
 
-            yield from (pickle.loads(lzma.decompress(row["raw_note"])) for row in cur)
+            for row in cur:
+                raw_note = self._get_raw_note(
+                    row["title"],
+                    row["guid"],
+                    row["raw_note"],
+                )
+
+                if raw_note:
+                    yield raw_note
 
     def get_notes_for_sync(self) -> Tuple[NoteForSync, ...]:
         with self.db as con:
@@ -375,6 +390,26 @@ def _get_notes_by_notebook(self, notebook_guid: str) -> List[str]:
 
             return [r["guid"] for r in sorted_notes]
 
+    def _get_raw_note(self, note_title: str, note_guid: str, raw_note: bytes):
+        try:
+            return pickle.loads(lzma.decompress(raw_note))
+        except Exception as e:
+            if logger.getEffectiveLevel() == logging.DEBUG:
+                logger.exception(f"Note '{note_title}' [{note_guid}] is corrupt: {e}")
+
+            logger.warning(
+                f"Note '{note_title}' [{note_guid}] is corrupt, it will be re-downloaded during next sync"
+            )
+
+            self._mark_note_for_redownload(note_guid)
+
+    def _mark_note_for_redownload(self, note_guid: str):
+        with self.db as con:
+            con.execute(
+                "update notes set raw_note=NULL, is_active=NULL where guid=?",
+                (note_guid,),
+            )
+
 
 class ConfigStorage(SqliteStorage):
     def set_config_value(self, name: str, config_value: str) -> None:

tests/test_note_storage.py

@@ -1,3 +1,4 @@
+import logging
 from pathlib import Path
 
 import pytest
@@ -341,6 +342,69 @@ def test_notes_trash(fake_storage):
     assert result_notes == expected_notes
 
 
+def test_notes_corrupt(fake_storage, caplog):
+    test_notes = [
+        Note(
+            guid="id1",
+            title="test",
+            content="test",
+            notebookGuid="notebook1",
+            active=True,
+        ),
+        Note(
+            guid="id2",
+            title="test",
+            content="test",
+            notebookGuid="notebook1",
+            active=True,
+        ),
+        Note(
+            guid="id3",
+            title="test",
+            content="test",
+            notebookGuid="notebook1",
+            active=True,
+        ),
+    ]
+
+    expected_notes = [
+        Note(
+            guid="id1",
+            title="test",
+            content="test",
+            notebookGuid="notebook1",
+            active=True,
+        ),
+        Note(
+            guid="id2",
+            title="test",
+            content="test",
+            notebookGuid="notebook1",
+            active=True,
+        ),
+    ]
+
+    for note in test_notes:
+        fake_storage.notes.add_note(note)
+
+    with fake_storage.db as con:
+        con.execute("UPDATE notes SET raw_note=? WHERE guid=?", (b"123", "id3"))
+
+    with caplog.at_level(logging.DEBUG, logger="evernote_backup"):
+        result_notes = list(fake_storage.notes.iter_notes("notebook1"))
+
+    result_notes_for_sync = fake_storage.notes.get_notes_for_sync()
+
+    assert result_notes == expected_notes
+    assert len(result_notes_for_sync) == 1
+    assert result_notes_for_sync[0].guid == "id3"
+    assert "Traceback" in caplog.text
+    assert (
+        caplog.messages[1]
+        == "Note 'test' [id3] is corrupt, it will be re-downloaded during next sync"
+    )
+
+
 def test_get_notes_for_sync(fake_storage):
     test_notes = [
         Note(