diff --git a/Dockerfile b/Dockerfile index f898dbcdd9d7ad66bc7769d0830a1a193fa39b08..f0e3eae325fdf71d58e4ef58a7d2e7763a1b8ddb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,5 +17,5 @@ to-static-jsonp COPY Pipfile Pipfile.lock /app/ RUN cd /app && pipenv install --system --deploy -COPY export_apkgs.py /app/export_apkgs.py -RUN echo "#!/bin/sh\n$(python-libfaketime)\n/app/export_apkgs.py \$@" > /bin/export_apkgs && chmod +x /bin/export_apkgs +COPY ./export_apkgs /app/export_apkgs +RUN echo "#!/bin/sh\n$(python-libfaketime)\npython3 -m /app/export_apkgs \$@" > /bin/export_apkgs && chmod +x /bin/export_apkgs diff --git a/export_apkgs/__init__.py b/export_apkgs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b04926f92f067a5c6e5938c41c25afd13a8e82c --- /dev/null +++ b/export_apkgs/__init__.py @@ -0,0 +1,17 @@ +from .config import load_apkg_spec +from .exporter import ApkgExporter +from pathlib import Path +from tempfile import TemporaryDirectory +import argparse + +def export_package_from_spec(spec_path: Path, args: argparse.Namespace) -> Path: + with TemporaryDirectory() as intermediate_dir: + package_directory = spec_path.parent + spec = load_apkg_spec(spec_path) + exporter = ApkgExporter( + spec=spec, + content_dir=package_directory, + intermediate_dir=Path(intermediate_dir), + args=args + ) + return exporter.export() diff --git a/export_apkgs/__main__.py b/export_apkgs/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..51163e6f4df4ea81368ecc7db15b9833257fc956 --- /dev/null +++ b/export_apkgs/__main__.py @@ -0,0 +1,21 @@ +from . import export_package_from_spec +from .config import APKG_SPEC_FILENAME +from pathlib import Path +import argparse + +parser = argparse.ArgumentParser( + prog="export_apkgs", + description="Generates apkgs from a spec in a specified directory") +parser.add_argument('-c', '--content', default='content', help="path to YAML or directory tree that contains .apkg-spec.yaml, each spec generates an APKG") +parser.add_argument('-t', '--templates-dir', required=True) +parser.add_argument('-o', '--output-dir', help="directory to put the output apkgs in") +parser.add_argument('-n', '--dry-run', action='store_true', required=False, help="Don't actually build Anki files, but print the files that would be generated") +args = parser.parse_args() +content = Path(args.content) +if content.suffix == '.yaml' or content.suffix == '.yml': + # directly specified a YAML as argument + export_package_from_spec(content, args) +else: + # got a directory or directory tree to find specs in + for apkg_spec_path in Path(args.content).glob(f'**/{APKG_SPEC_FILENAME}'): + export_package_from_spec(apkg_spec_path, args) diff --git a/export_apkgs/config.py b/export_apkgs/config.py new file mode 100644 index 0000000000000000000000000000000000000000..c0e5a3f34a7d55841748a8611004e4a69016a5ff --- /dev/null +++ b/export_apkgs/config.py @@ -0,0 +1,16 @@ +from .model import ApkgSpec, TemplateSpec +from pathlib import Path +from yaml import load, CLoader + +APKG_SPEC_FILENAME = '.apkg-spec.yaml' +TEMPLATE_SPEC_FILENAME = '.template-spec.yaml' + +def load_template_spec(templates_dir: Path, template: str) -> TemplateSpec: + with open(templates_dir / template / TEMPLATE_SPEC_FILENAME) as spec_yaml: + spec_yaml = load(spec_yaml, Loader=CLoader) + return TemplateSpec(**spec_yaml) + +def load_apkg_spec(spec_path: Path) -> ApkgSpec: + with open(spec_path) as spec_yaml: + spec_yaml = load(spec_yaml, Loader=CLoader) + return ApkgSpec(**spec_yaml) diff --git a/export_apkgs/error.py b/export_apkgs/error.py new file mode 100644 index 0000000000000000000000000000000000000000..a0b1667cca2566266e63472f2b17f336d97d5031 --- /dev/null +++ b/export_apkgs/error.py @@ -0,0 +1,2 @@ +class MissingResourceException(Exception): + pass diff --git a/export_apkgs.py b/export_apkgs/exporter.py old mode 100755 new mode 100644 similarity index 79% rename from export_apkgs.py rename to export_apkgs/exporter.py index eaedd2e26159a9c507ea5f421fd01e021a14a4b6..2358305314f51d71f1b2d109ae8e5d24e78cf78b --- a/export_apkgs.py +++ b/export_apkgs/exporter.py @@ -1,102 +1,22 @@ -#!/usr/bin/env python3 +from .config import load_template_spec +from .error import MissingResourceException +from .format import format_time +from .model import * from anki.collection import Collection, ExportAnkiPackageOptions, ImportAnkiPackageOptions, ImportAnkiPackageRequest from anki.import_export_pb2 import ImportAnkiPackageUpdateCondition from anki.models import ChangeNotetypeRequest, NotetypeDict from anki.notes import Note -from datetime import datetime, timezone +from datetime import datetime from functools import reduce from itertools import chain from libfaketime import fake_time -from pathlib import Path -from pydantic import BaseModel, field_serializer, field_validator -from semver import Version -from tempfile import TemporaryDirectory -from typing import Literal -from yaml import load, CLoader from os import getenv +from pathlib import Path import argparse import csv -APKG_SPEC_FILENAME = '.apkg-spec.yaml' -TEMPLATE_SPEC_FILENAME = '.template-spec.yaml' - -parser = argparse.ArgumentParser( - prog="export_apkgs.py", - description="Generates apkgs from a spec in a specified directory") -parser.add_argument('-c', '--content', default='content', help="path to YAML or directory tree that contains .apkg-spec.yaml, each spec generates an APKG") -parser.add_argument('-t', '--templates-dir', required=True) -parser.add_argument('-o', '--output-dir', help="directory to put the output apkgs in") -parser.add_argument('-n', '--dry-run', action='store_true', required=False, help="Don't actually build Anki files, but print the files that would be generated") - -class NoteType(BaseModel): - id: datetime - name: str - fields: list[str] - -class CardType(BaseModel): - name: str - template: Path - -class ImportApkg(BaseModel): - note_type: str - -class ImportCsv(BaseModel): - content_version: datetime - note_type: str - file_patterns: list[str]|None - deck_name_pattern: str - fields_mapping: list[str] - fields_static: dict[str, str] - tags: list[str] - -class ImportImages(BaseModel): - content_version: datetime - note_type: str - deck_name_pattern: str - fields_mapping: dict[str, str] - -class TemplateSpec(BaseModel): - template_version: datetime - note_type: NoteType - card_types: list[CardType] - resource_paths: list[Path] - -class ApkgSpec(BaseModel): - # semver of the content version, appended to the filename after a dash - content_version: str - templates: list[str] - content: list[ - dict[Literal['import_apkg'], ImportApkg]| - dict[Literal['import_csv'], ImportCsv]| - dict[Literal['import_images'], ImportImages] - ] - resource_paths: list[Path] - - @field_serializer('content_version') - def dump_content_version(self, ver: Version): - return str(ver) - - @field_validator('content_version') - def validate_content_version(cls, version): - Version.parse(version) - return version - - -def load_apkg_spec(spec_path: Path) -> ApkgSpec: - with open(spec_path) as spec_yaml: - spec_yaml = load(spec_yaml, Loader=CLoader) - return ApkgSpec(**spec_yaml) - -def load_template_spec(templates_dir: Path, template: str) -> TemplateSpec: - with open(templates_dir / template / TEMPLATE_SPEC_FILENAME) as spec_yaml: - spec_yaml = load(spec_yaml, Loader=CLoader) - return TemplateSpec(**spec_yaml) - -def format_time(time: datetime): - """ Formats the given time in a format suitable for faketime """ - return time.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') - TEMP_DECK_NAME = 'just_imported' + class ApkgExporter: spec: ApkgSpec content_dir: Path @@ -109,6 +29,12 @@ class ApkgExporter: added_note_type_ids: list[int] added_card_types: set[str] resource_pseudo_usages: str + """ + Note GUID against deck. + + Set from the deck field in CSV. + """ + note_deck_patterns: dict[str, str] def __init__(self, spec: ApkgSpec, content_dir: Path, intermediate_dir: Path, args: argparse.Namespace) -> None: self.spec = spec @@ -121,6 +47,7 @@ class ApkgExporter: self.added_note_type_ids = [] self.added_card_types = set() self.resource_pseudo_usages = '' + self.note_deck_patterns = {} def export(self) -> Path: @@ -205,6 +132,8 @@ class ApkgExporter: self.add_resource(col, resources_path) elif resources_path.is_dir(): self.add_resources(col, resources_path.glob('*')) + else: + raise MissingResourceException(f'resource {resources_path} not found or is not a file or directory') def add_resource(self, col: Collection, resource_path: Path): actual_path = col.media.add_file(resource_path) @@ -319,7 +248,6 @@ class ApkgExporter: if 'import_csv' in content: import_csv : ImportCsv = content['import_csv'] deck_name = import_csv.deck_name_pattern - fields_mapping = import_csv.fields_mapping ntid = name_to_ntid[import_csv.note_type] with fake_time(format_time(import_csv.content_version)): for pattern in import_csv.file_patterns or ['**/*.csv']: @@ -330,8 +258,7 @@ class ApkgExporter: tmp_deck, import_csv.tags, csv_path, - fields_mapping, - import_csv.fields_static) + import_csv) self.move_to_target_decks(col, tmp_deck, deck_name, csv_path) elif 'import_apkg' in content: import_apkg : ImportApkg = content['import_apkg'] @@ -377,23 +304,30 @@ class ApkgExporter: tmp_deck: int, tags: list[str], csv_path: Path, - fields_mapping: list[str], - fields_static: dict[str, str]): + opts: ImportCsv): with open(csv_path) as csv_file: - reader = csv.reader(csv_file, delimiter=';') + delimiter = opts.delimiter or "\t" + reader = csv.reader(csv_file, delimiter=delimiter) for row in reader: - note = Note(col, note_type_id) - note = self.new_note(col, row[fields_mapping.index('guid')], note_type_id) - for idx, field in enumerate(fields_mapping): - if idx >= len(row): - raise Exception(f'CSV row is missing {field} at index {idx}:\n{row}') - if field != 'guid': - note[field] = row[idx] - for field, value in fields_static.items(): - note[field] = value - for tag in tags: - note.add_tag(tag) - col.add_note(note, tmp_deck) + # hash lines are comments + if len(row) >= 1 and not row[0].startswith('#'): + note = Note(col, note_type_id) + guid = row[opts.fields_mapping.index('guid')] + note = self.new_note(col, guid, note_type_id) + for idx, field in enumerate(opts.fields_mapping): + if field != '': + if idx >= len(row): + raise Exception(f'CSV row is missing {field} at index {idx}:\n{row}') + # deck overrides the pattern per note + if field == 'deck': + self.note_deck_patterns[guid] = row[idx] + elif field != 'guid': + note[field] = row[idx] + for field, value in opts.fields_static.items(): + note[field] = value + for tag in tags: + note.add_tag(tag) + col.add_note(note, tmp_deck) def field_columns(self, col: Collection, note_type_id: int, mapping: list[str]) -> list[int]: """ @@ -462,7 +396,9 @@ class ApkgExporter: for card_id in col.find_cards('deck:"' + TEMP_DECK_NAME + '"'): card = col.get_card(card_id) card_type_name = card.template()["name"] - desired_deck_name = self.format_deck_name(deck_name_pattern, csv_path, card_type_name) + guid = card.note().guid + pattern = self.note_deck_patterns[guid] if guid in self.note_deck_patterns else deck_name_pattern + desired_deck_name = self.format_deck_name(pattern, csv_path, card_type_name) card.did = self.get_or_create_deck_id(col, desired_deck_name) col.update_card(card) @@ -517,26 +453,3 @@ class ApkgExporter: for key, value in replacements.items(): pattern = pattern.replace('{{' + key + '}}', value) return pattern - -def export_package_from_spec(spec_path: Path, args: argparse.Namespace) -> Path: - with TemporaryDirectory() as intermediate_dir: - package_directory = spec_path.parent - spec = load_apkg_spec(spec_path) - exporter = ApkgExporter( - spec=spec, - content_dir=package_directory, - intermediate_dir=Path(intermediate_dir), - args=args - ) - return exporter.export() - -if __name__ == "__main__": - args = parser.parse_args() - content = Path(args.content) - if content.suffix == '.yaml' or content.suffix == '.yml': - # directly specified a YAML as argument - export_package_from_spec(content, args) - else: - # got a directory or directory tree to find specs in - for apkg_spec_path in Path(args.content).glob(f'**/{APKG_SPEC_FILENAME}'): - export_package_from_spec(apkg_spec_path, args) diff --git a/export_apkgs/format.py b/export_apkgs/format.py new file mode 100644 index 0000000000000000000000000000000000000000..b9c4f143e54b1be16bac9b623f718113f239c248 --- /dev/null +++ b/export_apkgs/format.py @@ -0,0 +1,5 @@ +from datetime import datetime, timezone + +def format_time(time: datetime): + """ Formats the given time in a format suitable for faketime """ + return time.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') diff --git a/export_apkgs/model.py b/export_apkgs/model.py new file mode 100644 index 0000000000000000000000000000000000000000..3b362eac774a3d992055ec5173e613aaf5cafc14 --- /dev/null +++ b/export_apkgs/model.py @@ -0,0 +1,59 @@ +from datetime import datetime +from pathlib import Path +from pydantic import BaseModel, field_serializer, field_validator +from semver import Version +from typing import Literal + +class NoteType(BaseModel): + id: datetime + name: str + fields: list[str] + +class CardType(BaseModel): + name: str + template: Path + +class ImportApkg(BaseModel): + note_type: str + +class ImportCsv(BaseModel): + content_version: datetime + delimiter: str + note_type: str + file_patterns: list[str]|None + deck_name_pattern: str + fields_mapping: list[str] + fields_static: dict[str, str] + tags: list[str] + +class ImportImages(BaseModel): + content_version: datetime + note_type: str + deck_name_pattern: str + fields_mapping: dict[str, str] + +class TemplateSpec(BaseModel): + template_version: datetime + note_type: NoteType + card_types: list[CardType] + resource_paths: list[Path] + +class ApkgSpec(BaseModel): + # semver of the content version, appended to the filename after a dash + content_version: str + templates: list[str] + content: list[ + dict[Literal['import_apkg'], ImportApkg]| + dict[Literal['import_csv'], ImportCsv]| + dict[Literal['import_images'], ImportImages] + ] + resource_paths: list[Path] + + @field_serializer('content_version') + def dump_content_version(self, ver: Version): + return str(ver) + + @field_validator('content_version') + def validate_content_version(cls, version): + Version.parse(version) + return version diff --git a/test/fixtures/anki_text/.apkg-spec.yaml b/test/fixtures/anki_text/.apkg-spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ead10a8a44b45bb2bd0a7475994c45d7e507b6ff --- /dev/null +++ b/test/fixtures/anki_text/.apkg-spec.yaml @@ -0,0 +1,23 @@ +content_version: 1.0.0 + +templates: +- q_a + +content: +- import_csv: + content_version: 2024-01-19 19:00:00+00:00 + note_type: Q/A Testnotetype + file_patterns: + - '*.txt' + delimiter: "\t" + # Not needed because the CSV contains the deck name + deck_name_pattern: 'UNUSED' + fields_mapping: + - guid + - deck + - Question + - Answer + fields_static: {} + tags: [] + +resource_paths: [] diff --git a/test/fixtures/anki_text/Tapirs.txt b/test/fixtures/anki_text/Tapirs.txt new file mode 100644 index 0000000000000000000000000000000000000000..7b13e86517e847822f9d8bc899bfdeb3565b4686 --- /dev/null +++ b/test/fixtures/anki_text/Tapirs.txt @@ -0,0 +1,6 @@ +#separator:tab +#html:true +#guid column:1 +#deck column:1 +#tags column:4 +x,/Dh9c@Q^ Tapirs::Species Which tapir has a prominent mane? Tapirus terrestris diff --git a/test/fixtures/csv/.apkg-spec.yaml b/test/fixtures/csv/.apkg-spec.yaml index 0c13501f9f75bfdac9f03ed50a5c173cabd31b30..5c2d69583643429d5e0bc38a840d042519140dc1 100644 --- a/test/fixtures/csv/.apkg-spec.yaml +++ b/test/fixtures/csv/.apkg-spec.yaml @@ -9,6 +9,7 @@ content: note_type: Q/A Testnotetype file_patterns: - '*.csv' + delimiter: ';' # and to making one deck per card type deck_name_pattern: '{{card_type}}' fields_mapping: diff --git a/test/fixtures/csv_updated_content/.apkg-spec.yaml b/test/fixtures/csv_updated_content/.apkg-spec.yaml index 18e0338dcde5da076c9971e9ccbc825d98d3e8aa..1e15dd28dcb6263c087e0676e2e50521800642e3 100644 --- a/test/fixtures/csv_updated_content/.apkg-spec.yaml +++ b/test/fixtures/csv_updated_content/.apkg-spec.yaml @@ -9,6 +9,7 @@ content: note_type: Q/A Testnotetype file_patterns: - '*.csv' + delimiter: ';' # and to making one deck per card type deck_name_pattern: '{{card_type}}' fields_mapping: diff --git a/test/fixtures/fields_static/.apkg-spec.yaml b/test/fixtures/fields_static/.apkg-spec.yaml index 455ae6ead9a0cea526bb6c353842df6a16ef2045..b4b47f81ac879e8bb85132c097564029ffa23b18 100644 --- a/test/fixtures/fields_static/.apkg-spec.yaml +++ b/test/fixtures/fields_static/.apkg-spec.yaml @@ -9,6 +9,7 @@ content: note_type: Q/A Testnotetype file_patterns: - '*.csv' + delimiter: ';' # and to making one deck per card type deck_name_pattern: '{{card_type}}' fields_mapping: diff --git a/test/fixtures/resource_missing/.apkg-spec.yaml b/test/fixtures/resource_missing/.apkg-spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de7f1f4485bcee53e2629a710d8d33898f0427a5 --- /dev/null +++ b/test/fixtures/resource_missing/.apkg-spec.yaml @@ -0,0 +1,23 @@ +content_version: 1.0.0 + +templates: +- q_a + +content: +- import_csv: + content_version: 2024-01-19 19:00:00+00:00 + note_type: Q/A Testnotetype + file_patterns: + - '*.csv' + delimiter: ';' + # and to making one deck per card type + deck_name_pattern: '{{card_type}}' + fields_mapping: + - guid + - Question + - Answer + fields_static: {} + tags: [] + +resource_paths: +- missing.txt diff --git a/test/fixtures/resource_missing/content.csv b/test/fixtures/resource_missing/content.csv new file mode 100644 index 0000000000000000000000000000000000000000..3c98112adacf4a22058eda6fb0e3b4646fe61b1f --- /dev/null +++ b/test/fixtures/resource_missing/content.csv @@ -0,0 +1 @@ +test-csv-note-1;What is the scientific name of the only tapir living in Asia?;Tapirus indicus \ No newline at end of file diff --git a/test/test_export_apkgs.py b/test/test_export_apkgs.py index 7ff7af3471cfa0b9d8f5c7943633d843fcfa15d7..65ceaf959c92f6a1f18ac1157694e44a6a5b1150 100644 --- a/test/test_export_apkgs.py +++ b/test/test_export_apkgs.py @@ -1,7 +1,8 @@ from anki.collection import Collection, ImportAnkiPackageRequest, ImportAnkiPackageOptions from anki.import_export_pb2 import ImportAnkiPackageUpdateCondition -from export_apkgs import export_package_from_spec from dataclasses import dataclass +from export_apkgs import export_package_from_spec +from export_apkgs.error import MissingResourceException from pathlib import Path from tempfile import TemporaryDirectory import unittest @@ -17,6 +18,8 @@ CONTENT_PATH_ANKI = 'test/fixtures/anki' CONTENT_PATH_CSV = 'test/fixtures/csv' CONTENT_PATH_CSV_UPDATED_CONTENT = 'test/fixtures/csv_updated_content' CONTENT_PATH_FIELDS_STATIC = 'test/fixtures/fields_static' +CONTENT_PATH_ANKI_TEXT = 'test/fixtures/anki_text' +CONTENT_PATH_RESOURCE_MISSING = 'test/fixtures/resource_missing' TEMPLATES_PATH = 'test/fixtures/templates' TEMPLATES_PATH_UPDATED = 'test/fixtures/templates_updated' @@ -362,6 +365,62 @@ class TestExportApkgs(unittest.TestCase): ) + def test_anki_text(self): + """ + Anki text export with deck name + """ + with TemporaryDirectory() as temp_collection_dir: + col = Collection(str(Path(temp_collection_dir) / "test.anki2")) + with TemporaryDirectory() as first_export_dir: + args_first = MockArgs( + content=CONTENT_PATH_ANKI_TEXT, + templates_dir=TEMPLATES_PATH, + output_dir=first_export_dir, + dry_run=False) + package = export_package_from_spec( + Path(CONTENT_PATH_ANKI_TEXT) / '.apkg-spec.yaml', + args_first) + + # import the first time, everything is new + result1 = col.import_anki_package(ImportAnkiPackageRequest( + package_path=str(package), + options=ImportAnkiPackageOptions( + merge_notetypes=True, + update_notes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + update_notetypes=ImportAnkiPackageUpdateCondition.IMPORT_ANKI_PACKAGE_UPDATE_CONDITION_IF_NEWER, + with_scheduling=False, + with_deck_configs=False, + ) + )) + self.assertEqual(len(result1.log.new), 1) + self.assertEqual( + result1.log.new[0].fields, + # the empty one is resources + [ + 'Which tapir has a prominent mane?', + 'Tapirus terrestris', + '' + ] + ) + self.assertEqual( + len(col.find_cards('deck:"Tapirs::Species"')), + 1) + + def test_missing_resource(self): + """ + Referenced resource file does not exist + """ + with TemporaryDirectory() as first_export_dir: + args_first = MockArgs( + content=CONTENT_PATH_RESOURCE_MISSING, + templates_dir=TEMPLATES_PATH, + output_dir=first_export_dir, + dry_run=False) + self.assertRaises(MissingResourceException, lambda: export_package_from_spec( + Path(CONTENT_PATH_RESOURCE_MISSING) / '.apkg-spec.yaml', + args_first)) + + def find_template(col: Collection, name: str): template = None for name_and_id in col.models.all_names_and_ids(): diff --git a/version.txt b/version.txt index 4a36342fcab700951adb18ae7adc930997f6c3f4..1454f6ed4b751b190ba93ad2c721979cc0977fbc 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -3.0.0 +4.0.1