diff --git a/.gitignore b/.gitignore index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..bee8a64b79a99590d5303307144172cfe824fbf7 100644 --- a/.gitignore +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..93634ff6d910c0e062b1f2cba26c70075d1dbfa9 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,19 @@ +image: durcheinander/export-apkgs-test-runner:1.0.0 + +default: + cache: + paths: + - .venv + +stages: +- test + +test: + stage: test + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + when: never + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_COMMIT_TAG + script: + - make check diff --git a/Dockerfile b/Dockerfile index e6b30553886b918875bc05b650d4d56dee204695..15c903f41d4e5c80ff0feafd855a8495d09d90f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,6 @@ python3 \ python3-pip \ zip \ curl - RUN npm install --global \ yarn \ mudslide diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..eeba479f5484527b8cb732e3b595b43d51b19d83 --- /dev/null +++ b/Makefile @@ -0,0 +1,21 @@ +ifneq ($(wildcard /bin/export_apkgs),) +# no pipenv needed in container, deps are preinstalled globally +PYTHON := $(shell python-libfaketime | sed 's/export //') python3 +PYTHON_NEEDED := +else +# lazy evaluation in case faketime dep is not installed yet +PYTHON = PIPENV_VENV_IN_PROJECT=1 $(shell .venv/bin/python-libfaketime | sed 's/export //') pipenv run python +# when running outside the container, install python deps for tests if needed +PYTHON_NEEDED := .venv/.project +endif + +.PHONY: all +all: check + +.PHONY: check +check: $(PYTHON_NEEDED) + $(PYTHON) -m unittest discover test + +.venv/.project: Pipfile Pipfile.lock + PIPENV_VENV_IN_PROJECT=1 pipenv install + touch $@ diff --git a/export-apkgs-test-runner/Dockerfile b/export-apkgs-test-runner/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..9b6c82ad814440d04eac723e260dd5f394bb7ff8 --- /dev/null +++ b/export-apkgs-test-runner/Dockerfile @@ -0,0 +1,16 @@ +FROM debian:12.4 + +RUN apt-get update && apt-get install -y \ +faketime \ +git \ +make \ +nodejs \ +npm \ +pipenv \ +python3 \ +python3-pip \ +zip \ +curl +RUN npm install --global \ +yarn \ +mudslide diff --git a/test/fixtures/anki/.apkg-spec.yaml b/test/fixtures/anki/.apkg-spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92ef7488e7e8155afcb935ef23c3473690740a23 --- /dev/null +++ b/test/fixtures/anki/.apkg-spec.yaml @@ -0,0 +1,8 @@ +content_version: 0.0.1 + +templates: +- q_a + +content: +- import_apkg: + note_type: Q/A Testnotetype \ No newline at end of file diff --git a/test/fixtures/anki/Test.apkg b/test/fixtures/anki/Test.apkg new file mode 100644 index 0000000000000000000000000000000000000000..28a3614f2b228aa8ae1ba806c9914101a758cdcd Binary files /dev/null and b/test/fixtures/anki/Test.apkg differ diff --git a/test/fixtures/csv/.apkg-spec.yaml b/test/fixtures/csv/.apkg-spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..274a5ede4ea93c1613898b87eac36aba91f2cb35 --- /dev/null +++ b/test/fixtures/csv/.apkg-spec.yaml @@ -0,0 +1,18 @@ +content_version: 1.0.0 + +templates: +- q_a + +content: +- import_csv: + content_version: 2024-01-19 19:00:00+00:00 + note_type: Q/A Testnotetype + file_patterns: + - '*.csv' + # and to making one deck per card type + deck_name_pattern: '{{card_type}}' + fields_mapping: + - guid + - Question + - Answer + tags: [] \ No newline at end of file diff --git a/test/fixtures/csv/content.csv b/test/fixtures/csv/content.csv new file mode 100644 index 0000000000000000000000000000000000000000..3c98112adacf4a22058eda6fb0e3b4646fe61b1f --- /dev/null +++ b/test/fixtures/csv/content.csv @@ -0,0 +1 @@ +test-csv-note-1;What is the scientific name of the only tapir living in Asia?;Tapirus indicus \ No newline at end of file diff --git a/test/fixtures/csv_updated_content/.apkg-spec.yaml b/test/fixtures/csv_updated_content/.apkg-spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c20f61a2701c946e588aed80a1a36349bb9fd78e --- /dev/null +++ b/test/fixtures/csv_updated_content/.apkg-spec.yaml @@ -0,0 +1,18 @@ +content_version: 1.0.0 + +templates: +- q_a + +content: +- import_csv: + content_version: 2024-01-19 20:00:00+00:00 + note_type: Q/A Testnotetype + file_patterns: + - '*.csv' + # and to making one deck per card type + deck_name_pattern: '{{card_type}}' + fields_mapping: + - guid + - Question + - Answer + tags: [] \ No newline at end of file diff --git a/test/fixtures/csv_updated_content/content.csv b/test/fixtures/csv_updated_content/content.csv new file mode 100644 index 0000000000000000000000000000000000000000..d1f926732965ce42331922094e9d0e09843759b8 --- /dev/null +++ b/test/fixtures/csv_updated_content/content.csv @@ -0,0 +1 @@ +test-csv-note-1;What is the scientific name of the only tapir living in Asia?;Tapirus indicus, Acrocodia indica being an outdated synonym. \ No newline at end of file diff --git a/test/fixtures/templates/q_a/.template-spec.yaml b/test/fixtures/templates/q_a/.template-spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81ef8297f8199e9b21622015604b52f3468c861f --- /dev/null +++ b/test/fixtures/templates/q_a/.template-spec.yaml @@ -0,0 +1,14 @@ +template_version: 2024-01-20 18:00:00+00:00 + +note_type: + id: 2024-01-20 01:00:00+00:00 + name: Q/A Testnotetype + fields: + - Question + - Answer + +card_types: +- name: Q/A Testcardtype + template: q_a + +resource_paths: [] \ No newline at end of file diff --git a/test/fixtures/templates/q_a/q_a/back.html b/test/fixtures/templates/q_a/q_a/back.html new file mode 100644 index 0000000000000000000000000000000000000000..aaee1c407d25495a1d8bd80f1d7508047105aff8 --- /dev/null +++ b/test/fixtures/templates/q_a/q_a/back.html @@ -0,0 +1 @@ +{{Answer}} \ No newline at end of file diff --git a/test/fixtures/templates/q_a/q_a/front.html b/test/fixtures/templates/q_a/q_a/front.html new file mode 100644 index 0000000000000000000000000000000000000000..14cb9f28dbcf41273d713f9140dfd69fcfcf0ff4 --- /dev/null +++ b/test/fixtures/templates/q_a/q_a/front.html @@ -0,0 +1 @@ +{{Question}} \ No newline at end of file diff --git a/test/fixtures/templates_updated/q_a/.template-spec.yaml b/test/fixtures/templates_updated/q_a/.template-spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02ef56a8c9e806366f130c7bf6b46a211305aa4d --- /dev/null +++ b/test/fixtures/templates_updated/q_a/.template-spec.yaml @@ -0,0 +1,14 @@ +template_version: 2024-01-20 19:00:00+00:00 + +note_type: + id: 2024-01-20 01:00:00+00:00 + name: Q/A Testnotetype + fields: + - Question + - Answer + +card_types: +- name: Q/A Testcardtype + template: q_a + +resource_paths: [] diff --git a/test/fixtures/templates_updated/q_a/q_a/back.html b/test/fixtures/templates_updated/q_a/q_a/back.html new file mode 100644 index 0000000000000000000000000000000000000000..5cc04047c87ec3899621a9dd2ef1783bc121cee6 --- /dev/null +++ b/test/fixtures/templates_updated/q_a/q_a/back.html @@ -0,0 +1 @@ +Back: {{Answer}} \ No newline at end of file diff --git a/test/fixtures/templates_updated/q_a/q_a/front.html b/test/fixtures/templates_updated/q_a/q_a/front.html new file mode 100644 index 0000000000000000000000000000000000000000..51c00f17bd74b3f03674daf131ee9919167ac2fb --- /dev/null +++ b/test/fixtures/templates_updated/q_a/q_a/front.html @@ -0,0 +1 @@ +Front: {{Question}} \ No newline at end of file diff --git a/test/test_export_apkgs.py b/test/test_export_apkgs.py new file mode 100644 index 0000000000000000000000000000000000000000..03259b01a0edfffa8245cc4ad9cfd3335677170e --- /dev/null +++ b/test/test_export_apkgs.py @@ -0,0 +1,337 @@ +from anki.collection import Collection, ImportAnkiPackageRequest, ImportAnkiPackageOptions +from anki.import_export_pb2 import ImportAnkiPackageUpdateCondition +from export_apkgs import export_package_from_spec +from dataclasses import dataclass +from pathlib import Path +from tempfile import TemporaryDirectory +import unittest + +@dataclass +class MockArgs: + content: str + templates_dir: str + output_dir: str + dry_run: bool + +CONTENT_PATH_ANKI = 'test/fixtures/anki' +CONTENT_PATH_CSV = 'test/fixtures/csv' +CONTENT_PATH_CSV_UPDATED_CONTENT = 'test/fixtures/csv_updated_content' +TEMPLATES_PATH = 'test/fixtures/templates' +TEMPLATES_PATH_UPDATED = 'test/fixtures/templates_updated' + +class TestExportApkgs(unittest.TestCase): + def test_generate_once_and_reimport(self): + """ + Basic sanity check: if the exported package is imported twice, + everything is a duplicate and nothing should change. + """ + with TemporaryDirectory() as temp_collection_dir: + col = Collection(str(Path(temp_collection_dir) / "test.anki2")) + with TemporaryDirectory() as first_export_dir: + args_first = MockArgs( + content=CONTENT_PATH_CSV, + templates_dir=TEMPLATES_PATH, + output_dir=first_export_dir, + dry_run=False) + package = export_package_from_spec( + Path(CONTENT_PATH_CSV) / '.apkg-spec.yaml', + args_first) + + # import the first time, everything is new + result1 = col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + self.assertEqual(len(result1.log.new), 1) + self.assertEqual(len(result1.log.duplicate), 0) + + # now import again, nothing should change + result2 = col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + self.assertEqual(len(result2.log.duplicate) == 1 and len(result2.log.new), 0, + f'Expected the note to be recognized as duplicate.\nLog1:\n{result1.log}\nLog2:\n{result2.log}') + + def test_generate_twice_and_reimport(self): + """ + This checks that if we generate the package twice and import it twice, + then the second import will not change anything because the content is + supposed to be the same. + + If we introduce errors that lead to different note and card IDs being + generated, then this test will fail. + """ + with TemporaryDirectory() as temp_collection_dir: + col = Collection(str(Path(temp_collection_dir) / "test.anki2")) + with TemporaryDirectory() as first_export_dir: + args_first = MockArgs( + content=CONTENT_PATH_CSV, + templates_dir=TEMPLATES_PATH, + output_dir=first_export_dir, + dry_run=False) + with TemporaryDirectory() as second_export_dir: + args_second = MockArgs( + content=CONTENT_PATH_CSV, + templates_dir=TEMPLATES_PATH, + output_dir=second_export_dir, + dry_run=False) + spec_path = Path(CONTENT_PATH_CSV) / '.apkg-spec.yaml' + first_package = export_package_from_spec(spec_path, args_first) + second_package = export_package_from_spec(spec_path, args_second) + + # debug: uncomment to view for testing + # shutil.copy(first_package, './test-export-first.apkg.zip') + # shutil.copy(second_package, './test-export-second.apkg.zip') + + # import the first time, everything is new + result1 = col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(first_package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + self.assertEqual(len(result1.log.new), 1) + self.assertEqual(len(result1.log.duplicate), 0) + + # now import again, nothing should change + result2 = col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(second_package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + self.assertTrue( + len(result2.log.duplicate) == 1 and len(result2.log.new) == 0, + f'Expected second import to be a duplicate, but something else happened\nfirst log: {result1.log}\nsecond log:\n{result2.log}') + + def test_content_update_overwrites_previous_note(self): + """ + Tests that bumping the modification time will update notes from previous versions. + """ + with TemporaryDirectory() as temp_collection_dir: + col = Collection(str(Path(temp_collection_dir) / "test.anki2")) + with TemporaryDirectory() as first_export_dir: + args_first = MockArgs( + content=CONTENT_PATH_CSV, + templates_dir=TEMPLATES_PATH, + output_dir=first_export_dir, + dry_run=False) + with TemporaryDirectory() as second_export_dir: + args_second = MockArgs( + content=CONTENT_PATH_CSV_UPDATED_CONTENT, + templates_dir=TEMPLATES_PATH, + output_dir=second_export_dir, + dry_run=False) + spec_path_old = Path(f'{CONTENT_PATH_CSV}/.apkg-spec.yaml') + spec_path_new = Path(f'{CONTENT_PATH_CSV_UPDATED_CONTENT}/.apkg-spec.yaml') + first_package = export_package_from_spec(spec_path_old, args_first) + second_package = export_package_from_spec(spec_path_new, args_second) + + # import the old version version + result1 = col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(first_package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + self.assertEqual(len(result1.log.updated), 0) + self.assertEqual(len(result1.log.new), 1) + self.assertEqual(len(result1.log.duplicate), 0) + + # now the update + result2 = col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(second_package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + self.assertTrue(len(result2.log.updated) == 1 and len(result2.log.new) == 0 and len(result2.log.duplicate) == 0, + f'Expected the note to be recognized as update.\nLog1:\n{result1.log}\nLog2:\n{result2.log}') + + + def test_template_update_overwrites_previous_template_csv(self): + """ + Tests that bumping the modification time will update notes from previous versions + of CSV content. + """ + with TemporaryDirectory() as temp_collection_dir: + col = Collection(str(Path(temp_collection_dir) / "test.anki2")) + with TemporaryDirectory() as first_export_dir: + args_first = MockArgs( + content=CONTENT_PATH_CSV, + templates_dir=TEMPLATES_PATH, + output_dir=first_export_dir, + dry_run=False) + with TemporaryDirectory() as second_export_dir: + args_second = MockArgs( + content=CONTENT_PATH_CSV, + templates_dir=TEMPLATES_PATH_UPDATED, + output_dir=second_export_dir, + dry_run=False) + spec_path = Path(f'{CONTENT_PATH_CSV}/.apkg-spec.yaml') + first_package = export_package_from_spec(spec_path, args_first) + second_package = export_package_from_spec(spec_path, args_second) + + # debug: uncomment to view for testing + #shutil.copy(first_package, './test-export-first.apkg.zip') + #shutil.copy(second_package, './test-export-second.apkg.zip') + + # import the old version version + col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(first_package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + + template = find_template(col, 'Q/A Testnotetype') + self.assertIsNotNone(template, 'template not found in collection') + self.assertEqual( + template['qfmt'], + '{{Question}}') + self.assertEqual( + template['afmt'], + '{{Answer}}') + + # now the update + result = col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(second_package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + + # for some reason the change is only visible if we reload the collection + col.close() + col = Collection(str(Path(temp_collection_dir) / "test.anki2")) + + template = find_template(col, 'Q/A Testnotetype') + self.assertIsNotNone(template, 'template not found in collection') + self.assertEqual( + template['qfmt'], + 'Front: {{Question}}', + f'Template not updated successfully, log:\n{result.log}') + self.assertEqual( + template['afmt'], + 'Back: {{Answer}}') + + def test_template_update_overwrites_previous_template_anki(self): + """ + Tests that bumping the modification time will update notes from previous versions + of an imported anki package. + """ + with TemporaryDirectory() as temp_collection_dir: + col = Collection(str(Path(temp_collection_dir) / "test.anki2")) + with TemporaryDirectory() as first_export_dir: + args_first = MockArgs( + content=CONTENT_PATH_ANKI, + templates_dir=TEMPLATES_PATH, + output_dir=first_export_dir, + dry_run=False) + with TemporaryDirectory() as second_export_dir: + args_second = MockArgs( + content=CONTENT_PATH_ANKI, + templates_dir=TEMPLATES_PATH_UPDATED, + output_dir=second_export_dir, + dry_run=False) + spec_path = Path(f'{CONTENT_PATH_ANKI}/.apkg-spec.yaml') + first_package = export_package_from_spec(spec_path, args_first) + second_package = export_package_from_spec(spec_path, args_second) + + # debug: uncomment to view for testing + #shutil.copy(first_package, './test-export-first.apkg.zip') + #shutil.copy(second_package, './test-export-second.apkg.zip') + + # import the old version version + col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(first_package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + + template = find_template(col, 'Q/A Testnotetype') + self.assertIsNotNone(template, 'template not found in collection') + self.assertEqual( + template['qfmt'], + '{{Question}}') + self.assertEqual( + template['afmt'], + '{{Answer}}') + + # now the update + result = col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(second_package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + + # for some reason the change is only visible if we reload the collection + col.close() + col = Collection(str(Path(temp_collection_dir) / "test.anki2")) + + template = find_template(col, 'Q/A Testnotetype') + self.assertIsNotNone(template, 'template not found in collection') + self.assertEqual( + template['qfmt'], + 'Front: {{Question}}', + f'Template not updated successfully, log:\n{result.log}') + self.assertEqual( + template['afmt'], + 'Back: {{Answer}}') + +def find_template(col: Collection, name: str): + template = None + for name_and_id in col.models.all_names_and_ids(): + if name_and_id.name == name: + if template is None: + template = col.models.get(name_and_id.id)['tmpls'][0] + else: + raise Exception(f'Found more than one template with name {name}') + return template \ No newline at end of file