From 7197d2b1a6039e19af89038dc1c2b287dc061bb8 Mon Sep 17 00:00:00 2001
From: Philipp Stadler <hello@phstadler.com>
Date: Sun, 21 Jul 2024 16:02:10 +0200
Subject: [PATCH] refactor: split export_apkgs.py into a module
---
Dockerfile | 4 +-
export_apkgs/__init__.py | 17 +++
export_apkgs/__main__.py | 21 +++
export_apkgs/config.py | 16 +++
export_apkgs/error.py | 2 +
export_apkgs.py => export_apkgs/exporter.py | 120 ++----------------
export_apkgs/format.py | 5 +
export_apkgs/model.py | 59 +++++++++
.../fixtures/resource_missing/.apkg-spec.yaml | 23 ++++
test/fixtures/resource_missing/content.csv | 1 +
test/test_export_apkgs.py | 18 ++-
11 files changed, 172 insertions(+), 114 deletions(-)
create mode 100644 export_apkgs/__init__.py
create mode 100644 export_apkgs/__main__.py
create mode 100644 export_apkgs/config.py
create mode 100644 export_apkgs/error.py
rename export_apkgs.py => export_apkgs/exporter.py (84%)
mode change 100755 => 100644
create mode 100644 export_apkgs/format.py
create mode 100644 export_apkgs/model.py
create mode 100644 test/fixtures/resource_missing/.apkg-spec.yaml
create mode 100644 test/fixtures/resource_missing/content.csv
diff --git a/Dockerfile b/Dockerfile
index f898dbc..f0e3eae 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -17,5 +17,5 @@ to-static-jsonp
COPY Pipfile Pipfile.lock /app/
RUN cd /app && pipenv install --system --deploy
-COPY export_apkgs.py /app/export_apkgs.py
-RUN echo "#!/bin/sh\n$(python-libfaketime)\n/app/export_apkgs.py \$@" > /bin/export_apkgs && chmod +x /bin/export_apkgs
+COPY ./export_apkgs /app/export_apkgs
+RUN echo "#!/bin/sh\n$(python-libfaketime)\npython3 -m /app/export_apkgs \$@" > /bin/export_apkgs && chmod +x /bin/export_apkgs
diff --git a/export_apkgs/__init__.py b/export_apkgs/__init__.py
new file mode 100644
index 0000000..8b04926
--- /dev/null
+++ b/export_apkgs/__init__.py
@@ -0,0 +1,17 @@
+from .config import load_apkg_spec
+from .exporter import ApkgExporter
+from pathlib import Path
+from tempfile import TemporaryDirectory
+import argparse
+
+def export_package_from_spec(spec_path: Path, args: argparse.Namespace) -> Path:
+ with TemporaryDirectory() as intermediate_dir:
+ package_directory = spec_path.parent
+ spec = load_apkg_spec(spec_path)
+ exporter = ApkgExporter(
+ spec=spec,
+ content_dir=package_directory,
+ intermediate_dir=Path(intermediate_dir),
+ args=args
+ )
+ return exporter.export()
diff --git a/export_apkgs/__main__.py b/export_apkgs/__main__.py
new file mode 100644
index 0000000..51163e6
--- /dev/null
+++ b/export_apkgs/__main__.py
@@ -0,0 +1,21 @@
+from . import export_package_from_spec
+from .config import APKG_SPEC_FILENAME
+from pathlib import Path
+import argparse
+
+parser = argparse.ArgumentParser(
+ prog="export_apkgs",
+ description="Generates apkgs from a spec in a specified directory")
+parser.add_argument('-c', '--content', default='content', help="path to YAML or directory tree that contains .apkg-spec.yaml, each spec generates an APKG")
+parser.add_argument('-t', '--templates-dir', required=True)
+parser.add_argument('-o', '--output-dir', help="directory to put the output apkgs in")
+parser.add_argument('-n', '--dry-run', action='store_true', required=False, help="Don't actually build Anki files, but print the files that would be generated")
+args = parser.parse_args()
+content = Path(args.content)
+if content.suffix == '.yaml' or content.suffix == '.yml':
+ # directly specified a YAML as argument
+ export_package_from_spec(content, args)
+else:
+ # got a directory or directory tree to find specs in
+ for apkg_spec_path in Path(args.content).glob(f'**/{APKG_SPEC_FILENAME}'):
+ export_package_from_spec(apkg_spec_path, args)
diff --git a/export_apkgs/config.py b/export_apkgs/config.py
new file mode 100644
index 0000000..c0e5a3f
--- /dev/null
+++ b/export_apkgs/config.py
@@ -0,0 +1,16 @@
+from .model import ApkgSpec, TemplateSpec
+from pathlib import Path
+from yaml import load, CLoader
+
+APKG_SPEC_FILENAME = '.apkg-spec.yaml'
+TEMPLATE_SPEC_FILENAME = '.template-spec.yaml'
+
+def load_template_spec(templates_dir: Path, template: str) -> TemplateSpec:
+ with open(templates_dir / template / TEMPLATE_SPEC_FILENAME) as spec_yaml:
+ spec_yaml = load(spec_yaml, Loader=CLoader)
+ return TemplateSpec(**spec_yaml)
+
+def load_apkg_spec(spec_path: Path) -> ApkgSpec:
+ with open(spec_path) as spec_yaml:
+ spec_yaml = load(spec_yaml, Loader=CLoader)
+ return ApkgSpec(**spec_yaml)
diff --git a/export_apkgs/error.py b/export_apkgs/error.py
new file mode 100644
index 0000000..a0b1667
--- /dev/null
+++ b/export_apkgs/error.py
@@ -0,0 +1,2 @@
+class MissingResourceException(Exception):
+ pass
diff --git a/export_apkgs.py b/export_apkgs/exporter.py
old mode 100755
new mode 100644
similarity index 84%
rename from export_apkgs.py
rename to export_apkgs/exporter.py
index e0d9b7b..2358305
--- a/export_apkgs.py
+++ b/export_apkgs/exporter.py
@@ -1,103 +1,22 @@
-#!/usr/bin/env python3
+from .config import load_template_spec
+from .error import MissingResourceException
+from .format import format_time
+from .model import *
from anki.collection import Collection, ExportAnkiPackageOptions, ImportAnkiPackageOptions, ImportAnkiPackageRequest
from anki.import_export_pb2 import ImportAnkiPackageUpdateCondition
from anki.models import ChangeNotetypeRequest, NotetypeDict
from anki.notes import Note
-from datetime import datetime, timezone
+from datetime import datetime
from functools import reduce
from itertools import chain
from libfaketime import fake_time
-from pathlib import Path
-from pydantic import BaseModel, field_serializer, field_validator
-from semver import Version
-from tempfile import TemporaryDirectory
-from typing import Literal
-from yaml import load, CLoader
from os import getenv
+from pathlib import Path
import argparse
import csv
-APKG_SPEC_FILENAME = '.apkg-spec.yaml'
-TEMPLATE_SPEC_FILENAME = '.template-spec.yaml'
-
-parser = argparse.ArgumentParser(
- prog="export_apkgs.py",
- description="Generates apkgs from a spec in a specified directory")
-parser.add_argument('-c', '--content', default='content', help="path to YAML or directory tree that contains .apkg-spec.yaml, each spec generates an APKG")
-parser.add_argument('-t', '--templates-dir', required=True)
-parser.add_argument('-o', '--output-dir', help="directory to put the output apkgs in")
-parser.add_argument('-n', '--dry-run', action='store_true', required=False, help="Don't actually build Anki files, but print the files that would be generated")
-
-class NoteType(BaseModel):
- id: datetime
- name: str
- fields: list[str]
-
-class CardType(BaseModel):
- name: str
- template: Path
-
-class ImportApkg(BaseModel):
- note_type: str
-
-class ImportCsv(BaseModel):
- content_version: datetime
- delimiter: str
- note_type: str
- file_patterns: list[str]|None
- deck_name_pattern: str
- fields_mapping: list[str]
- fields_static: dict[str, str]
- tags: list[str]
-
-class ImportImages(BaseModel):
- content_version: datetime
- note_type: str
- deck_name_pattern: str
- fields_mapping: dict[str, str]
-
-class TemplateSpec(BaseModel):
- template_version: datetime
- note_type: NoteType
- card_types: list[CardType]
- resource_paths: list[Path]
-
-class ApkgSpec(BaseModel):
- # semver of the content version, appended to the filename after a dash
- content_version: str
- templates: list[str]
- content: list[
- dict[Literal['import_apkg'], ImportApkg]|
- dict[Literal['import_csv'], ImportCsv]|
- dict[Literal['import_images'], ImportImages]
- ]
- resource_paths: list[Path]
-
- @field_serializer('content_version')
- def dump_content_version(self, ver: Version):
- return str(ver)
-
- @field_validator('content_version')
- def validate_content_version(cls, version):
- Version.parse(version)
- return version
-
-
-def load_apkg_spec(spec_path: Path) -> ApkgSpec:
- with open(spec_path) as spec_yaml:
- spec_yaml = load(spec_yaml, Loader=CLoader)
- return ApkgSpec(**spec_yaml)
-
-def load_template_spec(templates_dir: Path, template: str) -> TemplateSpec:
- with open(templates_dir / template / TEMPLATE_SPEC_FILENAME) as spec_yaml:
- spec_yaml = load(spec_yaml, Loader=CLoader)
- return TemplateSpec(**spec_yaml)
-
-def format_time(time: datetime):
- """ Formats the given time in a format suitable for faketime """
- return time.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
-
TEMP_DECK_NAME = 'just_imported'
+
class ApkgExporter:
spec: ApkgSpec
content_dir: Path
@@ -213,6 +132,8 @@ class ApkgExporter:
self.add_resource(col, resources_path)
elif resources_path.is_dir():
self.add_resources(col, resources_path.glob('*'))
+ else:
+ raise MissingResourceException(f'resource {resources_path} not found or is not a file or directory')
def add_resource(self, col: Collection, resource_path: Path):
actual_path = col.media.add_file(resource_path)
@@ -532,26 +453,3 @@ class ApkgExporter:
for key, value in replacements.items():
pattern = pattern.replace('{{' + key + '}}', value)
return pattern
-
-def export_package_from_spec(spec_path: Path, args: argparse.Namespace) -> Path:
- with TemporaryDirectory() as intermediate_dir:
- package_directory = spec_path.parent
- spec = load_apkg_spec(spec_path)
- exporter = ApkgExporter(
- spec=spec,
- content_dir=package_directory,
- intermediate_dir=Path(intermediate_dir),
- args=args
- )
- return exporter.export()
-
-if __name__ == "__main__":
- args = parser.parse_args()
- content = Path(args.content)
- if content.suffix == '.yaml' or content.suffix == '.yml':
- # directly specified a YAML as argument
- export_package_from_spec(content, args)
- else:
- # got a directory or directory tree to find specs in
- for apkg_spec_path in Path(args.content).glob(f'**/{APKG_SPEC_FILENAME}'):
- export_package_from_spec(apkg_spec_path, args)
diff --git a/export_apkgs/format.py b/export_apkgs/format.py
new file mode 100644
index 0000000..b9c4f14
--- /dev/null
+++ b/export_apkgs/format.py
@@ -0,0 +1,5 @@
+from datetime import datetime, timezone
+
+def format_time(time: datetime):
+ """ Formats the given time in a format suitable for faketime """
+ return time.astimezone(timezone.utc).strftime('%Y-%m-%d %H:%M:%S')
diff --git a/export_apkgs/model.py b/export_apkgs/model.py
new file mode 100644
index 0000000..3b362ea
--- /dev/null
+++ b/export_apkgs/model.py
@@ -0,0 +1,59 @@
+from datetime import datetime
+from pathlib import Path
+from pydantic import BaseModel, field_serializer, field_validator
+from semver import Version
+from typing import Literal
+
+class NoteType(BaseModel):
+ id: datetime
+ name: str
+ fields: list[str]
+
+class CardType(BaseModel):
+ name: str
+ template: Path
+
+class ImportApkg(BaseModel):
+ note_type: str
+
+class ImportCsv(BaseModel):
+ content_version: datetime
+ delimiter: str
+ note_type: str
+ file_patterns: list[str]|None
+ deck_name_pattern: str
+ fields_mapping: list[str]
+ fields_static: dict[str, str]
+ tags: list[str]
+
+class ImportImages(BaseModel):
+ content_version: datetime
+ note_type: str
+ deck_name_pattern: str
+ fields_mapping: dict[str, str]
+
+class TemplateSpec(BaseModel):
+ template_version: datetime
+ note_type: NoteType
+ card_types: list[CardType]
+ resource_paths: list[Path]
+
+class ApkgSpec(BaseModel):
+ # semver of the content version, appended to the filename after a dash
+ content_version: str
+ templates: list[str]
+ content: list[
+ dict[Literal['import_apkg'], ImportApkg]|
+ dict[Literal['import_csv'], ImportCsv]|
+ dict[Literal['import_images'], ImportImages]
+ ]
+ resource_paths: list[Path]
+
+ @field_serializer('content_version')
+ def dump_content_version(self, ver: Version):
+ return str(ver)
+
+ @field_validator('content_version')
+ def validate_content_version(cls, version):
+ Version.parse(version)
+ return version
diff --git a/test/fixtures/resource_missing/.apkg-spec.yaml b/test/fixtures/resource_missing/.apkg-spec.yaml
new file mode 100644
index 0000000..de7f1f4
--- /dev/null
+++ b/test/fixtures/resource_missing/.apkg-spec.yaml
@@ -0,0 +1,23 @@
+content_version: 1.0.0
+
+templates:
+- q_a
+
+content:
+- import_csv:
+ content_version: 2024-01-19 19:00:00+00:00
+ note_type: Q/A Testnotetype
+ file_patterns:
+ - '*.csv'
+ delimiter: ';'
+ # and to making one deck per card type
+ deck_name_pattern: '{{card_type}}'
+ fields_mapping:
+ - guid
+ - Question
+ - Answer
+ fields_static: {}
+ tags: []
+
+resource_paths:
+- missing.txt
diff --git a/test/fixtures/resource_missing/content.csv b/test/fixtures/resource_missing/content.csv
new file mode 100644
index 0000000..3c98112
--- /dev/null
+++ b/test/fixtures/resource_missing/content.csv
@@ -0,0 +1 @@
+test-csv-note-1;What is the scientific name of the only tapir living in Asia?;Tapirus indicus
\ No newline at end of file
diff --git a/test/test_export_apkgs.py b/test/test_export_apkgs.py
index aeb5a36..65ceaf9 100644
--- a/test/test_export_apkgs.py
+++ b/test/test_export_apkgs.py
@@ -1,7 +1,8 @@
from anki.collection import Collection, ImportAnkiPackageRequest, ImportAnkiPackageOptions
from anki.import_export_pb2 import ImportAnkiPackageUpdateCondition
-from export_apkgs import export_package_from_spec
from dataclasses import dataclass
+from export_apkgs import export_package_from_spec
+from export_apkgs.error import MissingResourceException
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
@@ -18,6 +19,7 @@ CONTENT_PATH_CSV = 'test/fixtures/csv'
CONTENT_PATH_CSV_UPDATED_CONTENT = 'test/fixtures/csv_updated_content'
CONTENT_PATH_FIELDS_STATIC = 'test/fixtures/fields_static'
CONTENT_PATH_ANKI_TEXT = 'test/fixtures/anki_text'
+CONTENT_PATH_RESOURCE_MISSING = 'test/fixtures/resource_missing'
TEMPLATES_PATH = 'test/fixtures/templates'
TEMPLATES_PATH_UPDATED = 'test/fixtures/templates_updated'
@@ -403,6 +405,20 @@ class TestExportApkgs(unittest.TestCase):
self.assertEqual(
len(col.find_cards('deck:"Tapirs::Species"')),
1)
+
+ def test_missing_resource(self):
+ """
+ Referenced resource file does not exist
+ """
+ with TemporaryDirectory() as first_export_dir:
+ args_first = MockArgs(
+ content=CONTENT_PATH_RESOURCE_MISSING,
+ templates_dir=TEMPLATES_PATH,
+ output_dir=first_export_dir,
+ dry_run=False)
+ self.assertRaises(MissingResourceException, lambda: export_package_from_spec(
+ Path(CONTENT_PATH_RESOURCE_MISSING) / '.apkg-spec.yaml',
+ args_first))
def find_template(col: Collection, name: str):
--
GitLab