diff --git a/Dockerfile b/Dockerfile index f898dbcdd9d7ad66bc7769d0830a1a193fa39b08..f0e3eae325fdf71d58e4ef58a7d2e7763a1b8ddb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,5 +17,5 @@ to-static-jsonp COPY Pipfile Pipfile.lock /app/ RUN cd /app && pipenv install --system --deploy -COPY export_apkgs.py /app/export_apkgs.py -RUN echo "#!/bin/sh\n$(python-libfaketime)\n/app/export_apkgs.py \$@" > /bin/export_apkgs && chmod +x /bin/export_apkgs +COPY ./export_apkgs /app/export_apkgs +RUN echo "#!/bin/sh\n$(python-libfaketime)\npython3 -m /app/export_apkgs \$@" > /bin/export_apkgs && chmod +x /bin/export_apkgs diff --git a/export_apkgs/__init__.py b/export_apkgs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8b04926f92f067a5c6e5938c41c25afd13a8e82c --- /dev/null +++ b/export_apkgs/__init__.py @@ -0,0 +1,17 @@ +from .config import load_apkg_spec +from .exporter import ApkgExporter +from pathlib import Path +from tempfile import TemporaryDirectory +import argparse + +def export_package_from_spec(spec_path: Path, args: argparse.Namespace) -> Path: + with TemporaryDirectory() as intermediate_dir: + package_directory = spec_path.parent + spec = load_apkg_spec(spec_path) + exporter = ApkgExporter( + spec=spec, + content_dir=package_directory, + intermediate_dir=Path(intermediate_dir), + args=args + ) + return exporter.export() diff --git a/export_apkgs/__main__.py b/export_apkgs/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..51163e6f4df4ea81368ecc7db15b9833257fc956 --- /dev/null +++ b/export_apkgs/__main__.py @@ -0,0 +1,21 @@ +from . import export_package_from_spec +from .config import APKG_SPEC_FILENAME +from pathlib import Path +import argparse + +parser = argparse.ArgumentParser( + prog="export_apkgs", + description="Generates apkgs from a spec in a specified directory") +parser.add_argument('-c', '--content', default='content', help="path to YAML or directory tree that contains .apkg-spec.yaml, each spec generates an APKG") +parser.add_argument('-t', '--templates-dir', required=True) +parser.add_argument('-o', '--output-dir', help="directory to put the output apkgs in") +parser.add_argument('-n', '--dry-run', action='store_true', required=False, help="Don't actually build Anki files, but print the files that would be generated") +args = parser.parse_args() +content = Path(args.content) +if content.suffix == '.yaml' or content.suffix == '.yml': + # directly specified a YAML as argument + export_package_from_spec(content, args) +else: + # got a directory or directory tree to find specs in + for apkg_spec_path in Path(args.content).glob(f'**/{APKG_SPEC_FILENAME}'): + export_package_from_spec(apkg_spec_path, args) diff --git a/export_apkgs/config.py b/export_apkgs/config.py new file mode 100644 index 0000000000000000000000000000000000000000..c0e5a3f34a7d55841748a8611004e4a69016a5ff --- /dev/null +++ b/export_apkgs/config.py @@ -0,0 +1,16 @@ +from .model import ApkgSpec, TemplateSpec +from pathlib import Path +from yaml import load, CLoader + +APKG_SPEC_FILENAME = '.apkg-spec.yaml' +TEMPLATE_SPEC_FILENAME = '.template-spec.yaml' + +def load_template_spec(templates_dir: Path, template: str) -> TemplateSpec: + with open(templates_dir / template / TEMPLATE_SPEC_FILENAME) as spec_yaml: + spec_yaml = load(spec_yaml, Loader=CLoader) + return TemplateSpec(**spec_yaml) + +def load_apkg_spec(spec_path: Path) -> ApkgSpec: + with open(spec_path) as spec_yaml: + spec_yaml = load(spec_yaml, Loader=CLoader) + return ApkgSpec(**spec_yaml) diff --git a/export_apkgs/error.py b/export_apkgs/error.py new file mode 100644 index 0000000000000000000000000000000000000000..a0b1667cca2566266e63472f2b17f336d97d5031 --- /dev/null +++ b/export_apkgs/error.py @@ -0,0 +1,2 @@ +class MissingResourceException(Exception): + pass diff --git a/export_apkgs.py b/export_apkgs/exporter.py old mode 100755 new mode 100644 similarity index 84% rename from export_apkgs.py rename to export_apkgs/exporter.py index e0d9b7bba54fae92af942e24c834d54073682765..2358305314f51d71f1b2d109ae8e5d24e78cf78b --- a/export_apkgs.py +++ b/export_apkgs/exporter.py @@ -1,103 +1,22 @@ -#!/usr/bin/env python3 +from .config import load_template_spec +from .error import MissingResourceException +from .format import format_time +from .model import * from anki.collection import Collection, ExportAnkiPackageOptions, ImportAnkiPackageOptions, ImportAnkiPackageRequest from anki.import_export_pb2 import ImportAnkiPackageUpdateCondition from anki.models import ChangeNotetypeRequest, NotetypeDict from anki.notes import Note -from datetime import datetime, timezone +from datetime import datetime from functools import reduce from itertools import chain from libfaketime import fake_time -from pathlib import Path -from pydantic import BaseModel, field_serializer, field_validator -from semver import Version -from tempfile import TemporaryDirectory -from typing import Literal -from yaml import load, CLoader from os import getenv +from pathlib import Path import argparse import csv -APKG_SPEC_FILENAME = '.apkg-spec.yaml' -TEMPLATE_SPEC_FILENAME = '.template-spec.yaml' - -parser = argparse.ArgumentParser( - prog="export_apkgs.py", - description="Generates apkgs from a spec in a specified directory") -parser.add_argument('-c', '--content', default='content', help="path to YAML or directory tree that contains .apkg-spec.yaml, each spec generates an APKG") -parser.add_argument('-t', '--templates-dir', required=True) -parser.add_argument('-o', '--output-dir', help="directory to put the output apkgs in") -parser.add_argument('-n', '--dry-run', action='store_true', required=False, help="Don't actually build Anki files, but print the files that would be generated") - -class NoteType(BaseModel): - id: datetime - name: str - fields: list[str] - -class CardType(BaseModel): - name: str - template: Path - -class ImportApkg(BaseModel): - note_type: str - -class ImportCsv(BaseModel): - content_version: datetime - delimiter: str - note_type: str - file_patterns: list[str]|None - deck_name_pattern: str - fields_mapping: list[str] - fields_static: dict[str, str] - tags: list[str] - -class ImportImages(BaseModel): - content_version: datetime - note_type: str - deck_name_pattern: str - fields_mapping: dict[str, str] - -class TemplateSpec(BaseModel): - template_version: datetime - note_type: NoteType - card_types: list[CardType] - resource_paths: list[Path] - -class ApkgSpec(BaseModel): - # semver of the content version, appended to the filename after a dash - content_version: str - templates: list[str] - content: list[ - dict[Literal['import_apkg'], ImportApkg]| - dict[Literal['import_csv'], ImportCsv]| - dict[Literal['import_images'], ImportImages] - ] - resource_paths: list[Path] - - @field_serializer('content_version') - def dump_content_version(self, ver: Version): - return str(ver) - - @field_validator('content_version') - def validate_content_version(cls, version): - Version.parse(version) - return version - - -def load_apkg_spec(spec_path: Path) -> ApkgSpec: - with open(spec_path) as spec_yaml: - spec_yaml = load(spec_yaml, Loader=CLoader) - return ApkgSpec(**spec_yaml) - -def load_template_spec(templates_dir: Path, template: str) -> TemplateSpec: - with open(templates_dir / template / TEMPLATE_SPEC_FILENAME) as spec_yaml: - spec_yaml = load(spec_yaml, Loader=CLoader) - return TemplateSpec(**spec_yaml) - -def format_time(time: datetime): - """ Formats the given time in a format suitable for faketime """ - return time.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') - TEMP_DECK_NAME = 'just_imported' + class ApkgExporter: spec: ApkgSpec content_dir: Path @@ -213,6 +132,8 @@ class ApkgExporter: self.add_resource(col, resources_path) elif resources_path.is_dir(): self.add_resources(col, resources_path.glob('*')) + else: + raise MissingResourceException(f'resource {resources_path} not found or is not a file or directory') def add_resource(self, col: Collection, resource_path: Path): actual_path = col.media.add_file(resource_path) @@ -532,26 +453,3 @@ class ApkgExporter: for key, value in replacements.items(): pattern = pattern.replace('{{' + key + '}}', value) return pattern - -def export_package_from_spec(spec_path: Path, args: argparse.Namespace) -> Path: - with TemporaryDirectory() as intermediate_dir: - package_directory = spec_path.parent - spec = load_apkg_spec(spec_path) - exporter = ApkgExporter( - spec=spec, - content_dir=package_directory, - intermediate_dir=Path(intermediate_dir), - args=args - ) - return exporter.export() - -if __name__ == "__main__": - args = parser.parse_args() - content = Path(args.content) - if content.suffix == '.yaml' or content.suffix == '.yml': - # directly specified a YAML as argument - export_package_from_spec(content, args) - else: - # got a directory or directory tree to find specs in - for apkg_spec_path in Path(args.content).glob(f'**/{APKG_SPEC_FILENAME}'): - export_package_from_spec(apkg_spec_path, args) diff --git a/export_apkgs/format.py b/export_apkgs/format.py new file mode 100644 index 0000000000000000000000000000000000000000..b9c4f143e54b1be16bac9b623f718113f239c248 --- /dev/null +++ b/export_apkgs/format.py @@ -0,0 +1,5 @@ +from datetime import datetime, timezone + +def format_time(time: datetime): + """ Formats the given time in a format suitable for faketime """ + return time.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S') diff --git a/export_apkgs/model.py b/export_apkgs/model.py new file mode 100644 index 0000000000000000000000000000000000000000..3b362eac774a3d992055ec5173e613aaf5cafc14 --- /dev/null +++ b/export_apkgs/model.py @@ -0,0 +1,59 @@ +from datetime import datetime +from pathlib import Path +from pydantic import BaseModel, field_serializer, field_validator +from semver import Version +from typing import Literal + +class NoteType(BaseModel): + id: datetime + name: str + fields: list[str] + +class CardType(BaseModel): + name: str + template: Path + +class ImportApkg(BaseModel): + note_type: str + +class ImportCsv(BaseModel): + content_version: datetime + delimiter: str + note_type: str + file_patterns: list[str]|None + deck_name_pattern: str + fields_mapping: list[str] + fields_static: dict[str, str] + tags: list[str] + +class ImportImages(BaseModel): + content_version: datetime + note_type: str + deck_name_pattern: str + fields_mapping: dict[str, str] + +class TemplateSpec(BaseModel): + template_version: datetime + note_type: NoteType + card_types: list[CardType] + resource_paths: list[Path] + +class ApkgSpec(BaseModel): + # semver of the content version, appended to the filename after a dash + content_version: str + templates: list[str] + content: list[ + dict[Literal['import_apkg'], ImportApkg]| + dict[Literal['import_csv'], ImportCsv]| + dict[Literal['import_images'], ImportImages] + ] + resource_paths: list[Path] + + @field_serializer('content_version') + def dump_content_version(self, ver: Version): + return str(ver) + + @field_validator('content_version') + def validate_content_version(cls, version): + Version.parse(version) + return version diff --git a/test/fixtures/resource_missing/.apkg-spec.yaml b/test/fixtures/resource_missing/.apkg-spec.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de7f1f4485bcee53e2629a710d8d33898f0427a5 --- /dev/null +++ b/test/fixtures/resource_missing/.apkg-spec.yaml @@ -0,0 +1,23 @@ +content_version: 1.0.0 + +templates: +- q_a + +content: +- import_csv: + content_version: 2024-01-19 19:00:00+00:00 + note_type: Q/A Testnotetype + file_patterns: + - '*.csv' + delimiter: ';' + # and to making one deck per card type + deck_name_pattern: '{{card_type}}' + fields_mapping: + - guid + - Question + - Answer + fields_static: {} + tags: [] + +resource_paths: +- missing.txt diff --git a/test/fixtures/resource_missing/content.csv b/test/fixtures/resource_missing/content.csv new file mode 100644 index 0000000000000000000000000000000000000000..3c98112adacf4a22058eda6fb0e3b4646fe61b1f --- /dev/null +++ b/test/fixtures/resource_missing/content.csv @@ -0,0 +1 @@ +test-csv-note-1;What is the scientific name of the only tapir living in Asia?;Tapirus indicus \ No newline at end of file diff --git a/test/test_export_apkgs.py b/test/test_export_apkgs.py index aeb5a365cec736cac6644164ebe4aa3169bbe0cd..65ceaf959c92f6a1f18ac1157694e44a6a5b1150 100644 --- a/test/test_export_apkgs.py +++ b/test/test_export_apkgs.py @@ -1,7 +1,8 @@ from anki.collection import Collection, ImportAnkiPackageRequest, ImportAnkiPackageOptions from anki.import_export_pb2 import ImportAnkiPackageUpdateCondition -from export_apkgs import export_package_from_spec from dataclasses import dataclass +from export_apkgs import export_package_from_spec +from export_apkgs.error import MissingResourceException from pathlib import Path from tempfile import TemporaryDirectory import unittest @@ -18,6 +19,7 @@ CONTENT_PATH_CSV = 'test/fixtures/csv' CONTENT_PATH_CSV_UPDATED_CONTENT = 'test/fixtures/csv_updated_content' CONTENT_PATH_FIELDS_STATIC = 'test/fixtures/fields_static' CONTENT_PATH_ANKI_TEXT = 'test/fixtures/anki_text' +CONTENT_PATH_RESOURCE_MISSING = 'test/fixtures/resource_missing' TEMPLATES_PATH = 'test/fixtures/templates' TEMPLATES_PATH_UPDATED = 'test/fixtures/templates_updated' @@ -403,6 +405,20 @@ class TestExportApkgs(unittest.TestCase): self.assertEqual( len(col.find_cards('deck:"Tapirs::Species"')), 1) + + def test_missing_resource(self): + """ + Referenced resource file does not exist + """ + with TemporaryDirectory() as first_export_dir: + args_first = MockArgs( + content=CONTENT_PATH_RESOURCE_MISSING, + templates_dir=TEMPLATES_PATH, + output_dir=first_export_dir, + dry_run=False) + self.assertRaises(MissingResourceException, lambda: export_package_from_spec( + Path(CONTENT_PATH_RESOURCE_MISSING) / '.apkg-spec.yaml', + args_first)) def find_template(col: Collection, name: str):