From 2e363d78406aa998c6047be4d2cca376e0b79fad Mon Sep 17 00:00:00 2001 From: Ciro Silvano Date: Fri, 26 Sep 2025 18:18:03 +0200 Subject: [PATCH 1/5] chronicon-export: migrated code from recensio.policy Migrated code to send XML export over SFTP Refs. https://github.com/syslabcom/scrum/issues/3960 --- setup.py | 3 +- src/recensio/plone/browser/configure.zcml | 6 + src/recensio/plone/browser/export.py | 189 ++++++++ .../templates/export_container_contextless.pt | 13 + src/recensio/plone/configure.zcml | 5 + src/recensio/plone/export.py | 412 ++++++++++++++++++ src/recensio/plone/interfaces.py | 15 + 7 files changed, 642 insertions(+), 1 deletion(-) create mode 100644 src/recensio/plone/browser/export.py create mode 100644 src/recensio/plone/browser/templates/export_container_contextless.pt create mode 100644 src/recensio/plone/export.py diff --git a/setup.py b/setup.py index 84851687..87847d5e 100644 --- a/setup.py +++ b/setup.py @@ -53,10 +53,11 @@ "plone.app.discussion", "collective.vdexvocabulary", "collective.z3cform.datagridfield", + "paramiko-ng", "pypdf", "reportlab", "xlrd", - "z3c.jbot", + "z3c.jbot" ], extras_require={ "test": [ diff --git a/src/recensio/plone/browser/configure.zcml b/src/recensio/plone/browser/configure.zcml index 7c84c2bd..2a547207 100644 --- a/src/recensio/plone/browser/configure.zcml +++ b/src/recensio/plone/browser/configure.zcml @@ -93,6 +93,12 @@ layer="recensio.plone.interfaces.IRecensioPloneLayer" /> + + diff --git a/src/recensio/plone/browser/export.py b/src/recensio/plone/browser/export.py new file mode 100644 index 00000000..491a7b60 --- /dev/null +++ b/src/recensio/plone/browser/export.py @@ -0,0 +1,189 @@ +from datetime import date +from paramiko import SFTPClient +from paramiko import Transport +from paramiko.ssh_exception import SSHException +from plone import api +from Products.Five.browser import BrowserView +from Products.statusmessages.interfaces import IStatusMessage +from recensio.plone.config import REVIEW_TYPES +from recensio.plone.interfaces import IReview +from recensio.plone.export import LZAExporter +from recensio.plone.export import register_doi +from recensio.plone.interfaces import IRecensioExporter +from recensio.plone.controlpanel.settings import IRecensioSettings +from time import time +from zope.annotation.interfaces import IAnnotations +from zope.component import getFactoriesFor +from zope.component import getUtility +from zope.component.interfaces import IFactory + +import logging +import transaction + + +log = logging.getLogger(__name__) + +EXPORT_TIMESTAMP_KEY = "recensio.policy.metadata_export_timestamp" + + +class MetadataExport(BrowserView): + def __call__(self): + log.info("Starting export") + annotations = IAnnotations(self.context) + if self.request.get("force", False): + del annotations[EXPORT_TIMESTAMP_KEY] + timestamp = annotations.get(EXPORT_TIMESTAMP_KEY, 0) + if time() - timestamp < 2 * 60 * 60: # 2 hours + log.info("export already running, abort") + return "An export is already running, aborting" + + annotations[EXPORT_TIMESTAMP_KEY] = time() + transaction.commit() + + try: + exporters = [ + (name, factory()) for name, factory in getFactoriesFor(IRecensioExporter) + ] + exporters_to_run = [(name, e) for name, e in exporters if e.needs_to_run()] + except Exception as e: + log.exception(e) + del annotations[EXPORT_TIMESTAMP_KEY] + msg = "Error, aborting export: " + str(e) + log.error(msg) + return msg + + if not exporters_to_run: + del annotations[EXPORT_TIMESTAMP_KEY] + log.info("export finished, nothing to do") + return "Nothing to do, no exporter requested an export run." + + for issue_or_volume in self.issues_and_volumes(): + for review in self.reviews(issue_or_volume): + for name, exporter in exporters_to_run: + try: + exporter.add_review(review) + except Exception as e: + log.error( + "Error in {0} - {1}: {2}".format( + review.getId(), e.__class__.__name__, e + ) + ) + # Free memory after every issue and volume + transaction.commit() + statuses = [] + for name, exporter in exporters_to_run: + try: + status = exporter.export() + statuses.append((name, status)) + except Exception as e: + log.error( + "Error in {0} - {1}: {2}".format(name, e.__class__.__name__, e) + ) + + del annotations[EXPORT_TIMESTAMP_KEY] + transaction.commit() + log.info("export finished") + + return "
\n".join([name + ": " + str(status) for name, status in statuses]) + + def items(self, portal_type=(), context=None, depth=None): + pc = api.portal.get_tool("portal_catalog") + if context is None: + context = self.context + parent_path = dict(query="/".join(context.getPhysicalPath())) + if depth is not None: + parent_path["depth"] = depth + query = dict(review_state="published", portal_type=portal_type, path=parent_path) + results = pc(**query) + if None in results: + query["b_size"] = len(results) + results = pc(**query) + for item in results: + try: + yield item.getObject() + except AttributeError: + log.warn("Could not getObject: " + item.getPath()) + continue + + def issues_and_volumes(self): + return self.items(portal_type=("Issue", "Volume")) + + def reviews(self, issue): + return self.items(portal_type=REVIEW_TYPES, context=issue, depth=1) + + +class ChroniconExport(BrowserView): + @property + def filename(self): + prefix = api.portal.get_registry_record(name="recensio.plone.settings.xml_export_filename_prefix") + return "%s_%s_all.zip" % ( + prefix, + date.today().strftime("%d%m%y"), + ) + + def __call__(self): + host = api.portal.get_registry_record(name="recensio.plone.settings.xml_export_server") + username = api.portal.get_registry_record(name="recensio.plone.settings.xml_export_username") + password = api.portal.get_registry_record(name="recensio.plone.settings.xml_export_password") + if not host: + return "no host configured" + log.info("Starting XML export to sftp") + + exporter = getUtility(IFactory, name="chronicon_exporter")() + export_xml = exporter.get_export_obj(self.context) + if export_xml is None: + msg = "Could not get export file object: {0}".format( + exporter.export_filename + ) + log.error(msg) + return msg + + zipstream = export_xml.file + try: + transport = Transport((host, 2222)) + transport.connect(username=username, password=password) + sftp = SFTPClient.from_transport(transport) + attribs = sftp.putfo(zipstream.open(), self.filename) + except (IOError, SSHException) as ioe: + msg = "Export failed, {0}: {1}".format(ioe.__class__.__name__, ioe) + log.error(msg) + return msg + if attribs.st_size == zipstream.size: + msg = "Export successful" + log.info(msg) + return msg + else: + msg = "Export failed, {0}/{1} bytes transferred".format( + attribs.st_size, zipstream.size + ) + log.error(msg) + return msg + + +class DaraUpdate(BrowserView): + """Send metadata to da|ra, in effect registering the object's DOI or + updating its metadata if already registered.""" + + def __call__(self): + if self.request.get("REQUEST_METHOD") == "POST": + status, message = register_doi(self.context) + IStatusMessage(self.request).addStatusMessage(message, type=status) + self.request.response.redirect(self.context.absolute_url()) + + +class ResetLZAExportFlag(BrowserView): + def _reset_flag(self, context): + paths = [] + if IReview.providedBy(context): + self.exporter._set_exported(context, value=False) + paths.append("/".join(context.getPhysicalPath())) + for child in getattr(context, "objectValues", lambda: [])(): + paths += self._reset_flag(child) + return paths + + def __call__(self): + self.exporter = LZAExporter() + paths = self._reset_flag(self.context) + return u"Reset LZA export flag for the following objects:\n\n" + u"\n".join( + paths + ) diff --git a/src/recensio/plone/browser/templates/export_container_contextless.pt b/src/recensio/plone/browser/templates/export_container_contextless.pt new file mode 100644 index 00000000..f4de2945 --- /dev/null +++ b/src/recensio/plone/browser/templates/export_container_contextless.pt @@ -0,0 +1,13 @@ + + + + + + diff --git a/src/recensio/plone/configure.zcml b/src/recensio/plone/configure.zcml index 52b39bee..73d89d79 100644 --- a/src/recensio/plone/configure.zcml +++ b/src/recensio/plone/configure.zcml @@ -44,6 +44,11 @@ + + Date: Thu, 16 Apr 2026 19:29:22 +0200 Subject: [PATCH 2/5] Completed the takeover of old code that was left out during the Plone 6 migration --- CHANGES.rst | 4 +- docs/recensio-exports.md | 476 +++++++++++++++ src/recensio/plone/browser/configure.zcml | 103 +++- src/recensio/plone/browser/export.py | 75 ++- src/recensio/plone/browser/listing.py | 2 +- .../browser/templates/export_container.pt | 21 + .../templates/export_container_contextless.pt | 29 +- .../plone/browser/templates/export_rj.pt | 60 ++ .../plone/browser/templates/export_rj_dara.pt | 115 ++++ .../plone/browser/templates/export_rm.pt | 61 ++ .../plone/browser/templates/export_rm_dara.pt | 142 +++++ .../plone/browser/templates/search.pt | 530 +++++++++++----- src/recensio/plone/browser/xml.py | 174 ++++++ src/recensio/plone/config.py | 6 + src/recensio/plone/export.py | 233 +++---- src/recensio/plone/interfaces.py | 1 + src/recensio/plone/tests/test_exporters.py | 569 ++++++++++++++++++ .../registry/solr.xml | 2 +- 18 files changed, 2282 insertions(+), 321 deletions(-) create mode 100644 docs/recensio-exports.md create mode 100644 src/recensio/plone/browser/templates/export_container.pt create mode 100644 src/recensio/plone/browser/templates/export_rj.pt create mode 100644 src/recensio/plone/browser/templates/export_rj_dara.pt create mode 100644 src/recensio/plone/browser/templates/export_rm.pt create mode 100644 src/recensio/plone/browser/templates/export_rm_dara.pt create mode 100644 src/recensio/plone/browser/xml.py create mode 100644 src/recensio/plone/tests/test_exporters.py diff --git a/CHANGES.rst b/CHANGES.rst index f409a052..8cb208d7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,7 +5,9 @@ Changelog 2.1.2 (unreleased) ------------------ -- Nothing changed yet. +- Complete the XML export revival work. + `#3960 `_ + [pilz] 2.1.1 (2026-04-05) diff --git a/docs/recensio-exports.md b/docs/recensio-exports.md new file mode 100644 index 00000000..b68f71ee --- /dev/null +++ b/docs/recensio-exports.md @@ -0,0 +1,476 @@ +# Recensio export overview + +(generated by AI) + +This document summarizes the export-related code in: + +- `src/recensio/plone/export.py` +- `src/recensio/plone/browser/export.py` +- `src/recensio/plone/browser/templates/export_rm.pt` +- `src/recensio/plone/browser/templates/export_rj.pt` + +## High-level picture + +The export code has three layers: + +1. **Exporter classes** in `export.py` + - collect review data + - render XML or CSV + - store generated files in the portal root + - includes `ContainerExportView`, the minimal BrowserView used to render + per-issue XML (lives here to avoid a circular import with `browser/export.py`) +2. **Browser views** in `browser/export.py` + - trigger export-related actions + - upload an already generated export + - trigger DOI registration + - reset export flags +3. **Registry settings** in the control panel + - SFTP target for Chronicon uploads + - filename prefix for uploaded files + - DOI registration endpoint and credentials + +There is also an important dependency on review-specific XML views elsewhere in +the package: + +- `@@xml` +- `@@xml-lza` +- `@@xml-dara` + +The exporters do not build metadata themselves. They call those views on review +objects and package the results. + +## How the pieces hang together + +### Chronicon flow + +The Chronicon-related workflow is split into two steps: + +1. **Generate** a portal-root file object named `export_metadata_xml.zip` + using `ChroniconExporter` +2. **Upload** that existing ZIP to the remote Chronicon SFTP target using the + `@@chronicon-export` browser view + +So `@@chronicon-export` is an **upload view**, not the generator. + +### Generic bulk export flow + +`MetadataExport` in `browser/export.py` is a generic bulk-export runner. It: + +- finds exporter factories for `IRecensioExporter` +- filters them with `needs_to_run()` +- iterates over published `Issue` and `Volume` objects +- collects published reviews below them +- feeds each review into each exporter via `add_review()` +- calls `export()` on each exporter at the end + +In the **current package snapshot**, `MetadataExport` is exposed as a browser +view named `@@metadata-export` on the Plone site root. There are still no +visible ZCML registrations for the CSV or LZA exporters as `IRecensioExporter` +factories. + +In contrast, the package **does** register a named utility factory for the +Chronicon exporter: + +- `chronicon_exporter` -> `ChroniconExporterFactory` + +So the current wiring is clearest for: + +- the generic export runner `@@metadata-export` +- the Chronicon upload view `@@chronicon-export` +- the DOI update helper `@@dara_update` +- the LZA reset helper `@@reset-lza-export-flag` + +## `export.py` + +## `ContainerExportView` + +A minimal `BrowserView` that wraps the per-issue XML template: + +- template: `browser/templates/export_container_contextless.pt` +- called by `ChroniconExporter.finish_issue()` to render one XML file per issue + or volume +- lives in `export.py` (not in `browser/export.py`) so that `browser/export.py` + can safely import from `export.py` without creating a circular import + +## `BaseExporter` + +`BaseExporter` provides shared helpers: + +- `get_export_obj(portal)` + - looks up the generated export file object in the portal root +- `is_recent(export_xml_obj)` + - treats an existing export file as recent if it was modified less than + 6 days ago + +This is used by several exporters to decide whether a fresh export is needed. + +## `ChroniconExporter` + +Purpose: + +- export review metadata as a ZIP file with **one XML file per issue or volume** + +Important attributes: + +- template: `browser/templates/export_container_contextless.pt` +- stored portal-root file name: `export_metadata_xml.zip` +- per-review XML source: `@@xml` + +How it works: + +- reviews are added in issue/volume order with `add_review(review)` +- each review is rendered through `review.restrictedTraverse("@@xml")()` +- when the current issue/volume changes, `finish_issue()` renders one XML file + for that container and stores it in memory +- `export()` writes all gathered XML files into a ZIP archive and stores it as a + `File` object in the portal root + +Generated internal XML filenames look like: + +- `recensio___.xml` +- or `recensio__.xml` when there is no issue layer + +Concurrency / freshness behavior: + +- `needs_to_run()` returns false if a recent portal-root export already exists +- `running_export()` uses a temporary cache ZIP in the OS temp directory and + treats an export as in progress for up to 1 hour + +Return values from `export()` are status objects such as: + +- file created +- current file already exists +- export already in progress + +## `BVIDExporter` + +Purpose: + +- write a CSV mapping BVID values to review URLs + +Stored portal-root file: + +- `bvid_export.csv` + +Behavior: + +- includes only reviews with a BVID +- writes either the canonical URI or the normal absolute URL depending on the + review configuration + +## `MissingBVIDExporter` + +Purpose: + +- write a CSV listing reviews that still miss a BVID + +Stored portal-root file: + +- `missing_bvid.csv` + +Behavior: + +- stores `(UID, ISBN/ISSN)` pairs for reviews without a BVID + +## `LZAExporter` + +Purpose: + +- export metadata **and full text PDFs** for long-term archive / deposit use + +It subclasses `ChroniconExporter` and changes: + +- stored portal-root file: `export_lza_xml.zip` +- per-review XML source: `@@xml-lza` + +Additional behavior: + +- includes PDFs inside the ZIP next to the XML metadata +- marks each exported review with the annotation key `LZA_EXPORTED` +- skips reviews that were already exported before + +This makes LZA export **incremental** by default. + +## `DaraExporter` + +Purpose: + +- collect `@@xml-dara` output for DOI registration-related export + +Current state: + +- the class exists, but `export()` is still a stub (`pass`) + +So it looks like a scaffold or incomplete path rather than an active export +implementation. + +## `register_doi_raw()` and `register_doi()` + +These functions handle DOI registration / metadata update against the configured +da|ra endpoint. + +They use registry settings for: + +- `doi_registration_url` +- `doi_registration_username` +- `doi_registration_password` + +Behavior: + +- build XML from `@@xml-dara` +- send it via HTTP Basic Auth +- translate HTTP/IO/value errors into user-facing status/message pairs + +Typical successful messages: + +- `DOI successfully registered` +- `Metadata updated` + +## `browser/export.py` + +This module contains only **browser views**. It imports `LZAExporter` and +`register_doi` from `export.py`; the dependency is one-directional. + +## `MetadataExport` + +Purpose: + +- generic bulk-runner for all registered `IRecensioExporter` exporters + +What it does: + +- uses a portal annotation key + `recensio.policy.metadata_export_timestamp` + as a coarse lock +- refuses to start if an export has run within the last 2 hours +- accepts a `force` request parameter to clear that lock first +- walks published issues/volumes and published reviews below them +- calls exporter `add_review()` for each review +- commits the transaction after each issue/volume to reduce memory pressure +- calls `export()` on each exporter and returns a HTML `
`-joined summary + +Registration: + +- ZCML name: `metadata-export` +- context: `Products.CMFPlone.interfaces.IPloneSiteRoot` +- permission: `zope2.View` + +Usage URL: + +```text +https://example.org/Plone/@@metadata-export +``` + +Notes: + +- this is the browser entry point that orchestrates the bulk export run +- it supports a `force` request parameter to clear the in-progress timestamp + guard before starting + +## `@@chronicon-export` + +Purpose: + +- upload the current Chronicon ZIP from the portal root to a remote SFTP server + +Registration: + +- ZCML name: `chronicon-export` +- context: `Products.CMFPlone.interfaces.IPloneSiteRoot` +- permission: `cmf.ManagePortal` + +Usage URL: + +```text +https://example.org/Plone/@@chronicon-export +``` + +Configuration used: + +- `xml_export_server` +- `xml_export_username` +- `xml_export_password` +- `xml_export_filename_prefix` + +Behavior: + +1. read SFTP settings from the registry +2. look up `export_metadata_xml.zip` in the portal root +3. open an SFTP connection to the configured host on **port 2222** +4. upload the ZIP +5. name the remote file as: + +```text +[prefix]_[DDMMYY]_all.zip +``` + +Example: + +```text +recensio_050426_all.zip +``` + +Important limitation: + +- this view only uploads an already prepared ZIP +- it does **not** regenerate `export_metadata_xml.zip` + +Typical return values: + +- `no host configured` +- `Could not get export file object: export_metadata_xml.zip` +- `Export successful` +- `Export failed, IOError: ...` +- `Export failed, SSHException: ...` +- `Export failed, / bytes transferred` + +## `DaraUpdate` + +Purpose: + +- register or update DOI metadata for the current object through da|ra + +Behavior: + +- only performs work on `POST` +- calls `register_doi(self.context)` +- stores the result as a Plone status message +- redirects back to the current object afterwards + +Registration: + +- ZCML name: `dara_update` +- context: `recensio.plone.interfaces.IReview` +- permission: `cmf.ModifyPortalContent` + +Usage pattern: + +- intended for review objects +- only does work on `POST` +- redirects back to the review after execution + +## `ResetLZAExportFlag` + +Purpose: + +- recursively clear the `LZA_EXPORTED` annotation so objects can be exported + again by `LZAExporter` + +Behavior: + +- walks the current context recursively +- for any object providing `IReview`, resets the LZA export flag to `False` +- returns a plain text list of affected object paths + +Registration: + +- ZCML name: `reset-lza-export-flag` +- context: `*` +- permission: `cmf.ModifyPortalContent` + +Usage pattern: + +- can be called on a subtree to clear LZA export flags recursively +- returns a plain text list of affected paths + +## Control panel settings involved + +The export code depends on these Recensio settings: + +### Chronicon / SFTP + +- `xml_export_server` +- `xml_export_username` +- `xml_export_password` +- `xml_export_filename_prefix` + +### DOI / da|ra + +- `doi_prefix` +- `doi_registration_url` +- `doi_registration_username` +- `doi_registration_password` + +## Practical summary + +If you want to run the export generation and Chronicon upload path, the +practical flow is: + +1. call `@@metadata-export` on the site root to generate or refresh export + artifacts +2. ensure the Chronicon SFTP settings are configured +3. call `@@chronicon-export` on the site root as a manager + +If you are working on the broader export subsystem, keep in mind: + +- `ChroniconExporter`, `LZAExporter`, BVID exporters, and DOI helpers live in + `export.py` +- `ContainerExportView` also lives in `export.py` to avoid a circular import +- `MetadataExport` is the bulk orchestration layer and is now exposed as + `@@metadata-export` +- `ChroniconExport` handles only the remote SFTP upload step +- `DaraUpdate` and `ResetLZAExportFlag` are also exposed as browser views +- some exporter classes still exist in code without obvious factory + registrations in the current package ZCML + +## Export templates + +### `export_rm.pt` — review of monograph + +Used for reviews of books/monographs. Fields are read directly as Dexterity +attributes (Python 3 / Plone 6 — no AT getter methods): + +| XML element | Context attribute | Behavior | +|---|---|---| +| `isbn` | `context/isbn` | `IBookReview` | +| `language_reviewed_object` | `context.languageReviewedText` | `ITextReview` | +| `pages` | `context/pages` | `ISerial` | +| `place_of_publication` | `context/placeOfPublication` | `IPrintedReview` | +| `publishing_company` | `context/publisher` | `IPrintedReview` | +| `series` | `context/series` | `ISerial` | +| `subtitle` | `context/subtitle` | `IBookReview` | +| `volume_number` | `context/seriesVol` | `ISerial` | +| `year` | `context/yearOfPublication` | `IPrintedReview` | +| `bvid` | `context/bv` | `IBase` | +| `classification_place` | `context.ddcPlace` | `IBase` | +| `classification_subject` | `context.ddcSubject` | `IBase` | +| `classification_time` | `context.ddcTime` | `IBase` | +| `doi` | `context/doi` (condition: `doiRegistrationActive`) | `IBaseReview` / `ISettingsDoi` | +| `language_review` | `context.languageReview` | `IBase` | +| `reviewer` | `context/reviewAuthors` (RelationList) → `author/to_object/lastname` | `IBase` | +| `page_first` | `context/pageStartOfReviewInJournal` | `IPagesOfReviewInJournal` | +| `page_last` | `context/pageEndOfReviewInJournal` | `IPagesOfReviewInJournal` | + +### `export_rj.pt` — review of journal + +Used for reviews of journals / journal articles. Follows the same Dexterity +attribute pattern as `export_rm.pt`. Additional journal-specific fields: + +| XML element | Context attribute | Behavior / field | +|---|---|---| +| `reviewed_journal_editor` | `context/editor` | content type field | +| `reviewed_journal_issn` | `context/issn` | `IJournalReview` | +| `reviewed_journal_issue` | `context/issueNumber` | `IJournalReview` | +| `reviewed_journal_publisher` | `context/publisher` | `IPrintedReview` | +| `reviewed_journal_pubplace` | `context/placeOfPublication` | `IPrintedReview` | +| `reviewed_journal_pubyear_real` | `context/yearOfPublication` | `IPrintedReview` | +| `reviewed_journal_pubyear_fictive` | `context/officialYearOfPublication` | `IJournalReview` | +| `reviewed_journal_title_short` | `context/shortnameJournal` | `IJournalReview` | +| `reviewed_journal_volume` | `context/volumeNumber` | `IJournalReview` | + +The `reviewer` block, `bvid`, `doi`, classification fields, `page_first` / +`page_last`, `filename`, `fulltext`, and `originalurl` are identical to +`export_rm.pt`. + +### Note on `reviewAuthors` + +`reviewAuthors` (defined on `IBase`) is a Dexterity `RelationList` of +`RelationValue` objects pointing to `Person` content items. Templates must go +through `to_object` to reach the actual person: + +```xml + + + + +``` diff --git a/src/recensio/plone/browser/configure.zcml b/src/recensio/plone/browser/configure.zcml index f27ca2bd..2d5ab578 100644 --- a/src/recensio/plone/browser/configure.zcml +++ b/src/recensio/plone/browser/configure.zcml @@ -122,13 +122,114 @@ layer="recensio.plone.interfaces.IRecensioPloneLayer" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/recensio/plone/browser/export.py b/src/recensio/plone/browser/export.py index 491a7b60..a16f947e 100644 --- a/src/recensio/plone/browser/export.py +++ b/src/recensio/plone/browser/export.py @@ -3,46 +3,48 @@ from paramiko import Transport from paramiko.ssh_exception import SSHException from plone import api +from plone.protect.interfaces import IDisableCSRFProtection from Products.Five.browser import BrowserView from Products.statusmessages.interfaces import IStatusMessage -from recensio.plone.config import REVIEW_TYPES -from recensio.plone.interfaces import IReview +from recensio.plone.config import REVIEW_TYPES_TO_EXPORT from recensio.plone.export import LZAExporter from recensio.plone.export import register_doi from recensio.plone.interfaces import IRecensioExporter -from recensio.plone.controlpanel.settings import IRecensioSettings +from recensio.plone.interfaces import IReview from time import time from zope.annotation.interfaces import IAnnotations from zope.component import getFactoriesFor from zope.component import getUtility from zope.component.interfaces import IFactory +from zope.interface import alsoProvides import logging -import transaction log = logging.getLogger(__name__) -EXPORT_TIMESTAMP_KEY = "recensio.policy.metadata_export_timestamp" +EXPORT_TIMESTAMP_KEY = "recensio.plone.metadata_export_timestamp" class MetadataExport(BrowserView): def __call__(self): + alsoProvides(self.request, IDisableCSRFProtection) log.info("Starting export") annotations = IAnnotations(self.context) if self.request.get("force", False): - del annotations[EXPORT_TIMESTAMP_KEY] + if EXPORT_TIMESTAMP_KEY in annotations: + del annotations[EXPORT_TIMESTAMP_KEY] timestamp = annotations.get(EXPORT_TIMESTAMP_KEY, 0) if time() - timestamp < 2 * 60 * 60: # 2 hours log.info("export already running, abort") return "An export is already running, aborting" annotations[EXPORT_TIMESTAMP_KEY] = time() - transaction.commit() try: exporters = [ - (name, factory()) for name, factory in getFactoriesFor(IRecensioExporter) + (name, factory()) + for name, factory in getFactoriesFor(IRecensioExporter) ] exporters_to_run = [(name, e) for name, e in exporters if e.needs_to_run()] except Exception as e: @@ -64,24 +66,16 @@ def __call__(self): exporter.add_review(review) except Exception as e: log.error( - "Error in {0} - {1}: {2}".format( + "Error in {} - {}: {}".format( review.getId(), e.__class__.__name__, e ) ) - # Free memory after every issue and volume - transaction.commit() statuses = [] for name, exporter in exporters_to_run: - try: - status = exporter.export() - statuses.append((name, status)) - except Exception as e: - log.error( - "Error in {0} - {1}: {2}".format(name, e.__class__.__name__, e) - ) + status = exporter.export() + statuses.append((name, status)) del annotations[EXPORT_TIMESTAMP_KEY] - transaction.commit() log.info("export finished") return "
\n".join([name + ": " + str(status) for name, status in statuses]) @@ -93,7 +87,9 @@ def items(self, portal_type=(), context=None, depth=None): parent_path = dict(query="/".join(context.getPhysicalPath())) if depth is not None: parent_path["depth"] = depth - query = dict(review_state="published", portal_type=portal_type, path=parent_path) + query = dict( + review_state="published", portal_type=portal_type, path=parent_path + ) results = pc(**query) if None in results: query["b_size"] = len(results) @@ -109,22 +105,31 @@ def issues_and_volumes(self): return self.items(portal_type=("Issue", "Volume")) def reviews(self, issue): - return self.items(portal_type=REVIEW_TYPES, context=issue, depth=1) + return self.items(portal_type=REVIEW_TYPES_TO_EXPORT, context=issue, depth=1) class ChroniconExport(BrowserView): @property def filename(self): - prefix = api.portal.get_registry_record(name="recensio.plone.settings.xml_export_filename_prefix") - return "%s_%s_all.zip" % ( + prefix = api.portal.get_registry_record( + name="recensio.plone.settings.xml_export_filename_prefix" + ) + return "{}_{}_all.zip".format( prefix, date.today().strftime("%d%m%y"), ) def __call__(self): - host = api.portal.get_registry_record(name="recensio.plone.settings.xml_export_server") - username = api.portal.get_registry_record(name="recensio.plone.settings.xml_export_username") - password = api.portal.get_registry_record(name="recensio.plone.settings.xml_export_password") + alsoProvides(self.request, IDisableCSRFProtection) + host = api.portal.get_registry_record( + name="recensio.plone.settings.xml_export_server" + ) + username = api.portal.get_registry_record( + name="recensio.plone.settings.xml_export_username" + ) + password = api.portal.get_registry_record( + name="recensio.plone.settings.xml_export_password" + ) if not host: return "no host configured" log.info("Starting XML export to sftp") @@ -132,28 +137,32 @@ def __call__(self): exporter = getUtility(IFactory, name="chronicon_exporter")() export_xml = exporter.get_export_obj(self.context) if export_xml is None: - msg = "Could not get export file object: {0}".format( + msg = "Could not get export file object: {}".format( exporter.export_filename ) log.error(msg) return msg zipstream = export_xml.file + transport = None try: transport = Transport((host, 2222)) transport.connect(username=username, password=password) sftp = SFTPClient.from_transport(transport) attribs = sftp.putfo(zipstream.open(), self.filename) - except (IOError, SSHException) as ioe: - msg = "Export failed, {0}: {1}".format(ioe.__class__.__name__, ioe) + except (OSError, SSHException) as ioe: + msg = f"Export failed, {ioe.__class__.__name__}: {ioe}" log.error(msg) return msg + finally: + if transport is not None: + transport.close() if attribs.st_size == zipstream.size: msg = "Export successful" log.info(msg) return msg else: - msg = "Export failed, {0}/{1} bytes transferred".format( + msg = "Export failed, {}/{} bytes transferred".format( attribs.st_size, zipstream.size ) log.error(msg) @@ -165,6 +174,7 @@ class DaraUpdate(BrowserView): updating its metadata if already registered.""" def __call__(self): + alsoProvides(self.request, IDisableCSRFProtection) if self.request.get("REQUEST_METHOD") == "POST": status, message = register_doi(self.context) IStatusMessage(self.request).addStatusMessage(message, type=status) @@ -182,8 +192,7 @@ def _reset_flag(self, context): return paths def __call__(self): + alsoProvides(self.request, IDisableCSRFProtection) self.exporter = LZAExporter() paths = self._reset_flag(self.context) - return u"Reset LZA export flag for the following objects:\n\n" + u"\n".join( - paths - ) + return "Reset LZA export flag for the following objects:\n\n" + "\n".join(paths) diff --git a/src/recensio/plone/browser/listing.py b/src/recensio/plone/browser/listing.py index d7df71ca..a7c33381 100644 --- a/src/recensio/plone/browser/listing.py +++ b/src/recensio/plone/browser/listing.py @@ -79,7 +79,7 @@ def filter_query(self, query): Solr indexes across sibling sites recensio, regio, altertum. We should return results only from the current site. """ - query = super(RecensioSearch, self).filter_query(query) + query = super().filter_query(query) if "path" not in query and "parent_path" not in query: query["path_parents"] = "/".join(api.portal.get().getPhysicalPath()) return query diff --git a/src/recensio/plone/browser/templates/export_container.pt b/src/recensio/plone/browser/templates/export_container.pt new file mode 100644 index 00000000..2bdb90fe --- /dev/null +++ b/src/recensio/plone/browser/templates/export_container.pt @@ -0,0 +1,21 @@ + + + + + + diff --git a/src/recensio/plone/browser/templates/export_container_contextless.pt b/src/recensio/plone/browser/templates/export_container_contextless.pt index f4de2945..8c991c2a 100644 --- a/src/recensio/plone/browser/templates/export_container_contextless.pt +++ b/src/recensio/plone/browser/templates/export_container_contextless.pt @@ -1,13 +1,20 @@ - - - - + + + + + diff --git a/src/recensio/plone/browser/templates/export_rj.pt b/src/recensio/plone/browser/templates/export_rj.pt new file mode 100644 index 00000000..a6473927 --- /dev/null +++ b/src/recensio/plone/browser/templates/export_rj.pt @@ -0,0 +1,60 @@ + + + Blažek + David + + + Institut der Akademie der Wissenschaften der Tschechischen Republik + 1210-9029! + 2! + !Slovanský ústav AV ČR + !Praha + !2009 + 2010 + !Germanoslavica. Zeitschrift für germano-slawische Studien + Germanoslavica + !20 + !English, Deutsch + + + Slowenien, Polen, Lettland, Deutschland, Tschechien + Sprachgeschichte, Wirtschaftsgeschichte, Literaturgeschichte, Siedlungs-, Stadt- und Ortsgeschichte, Bildungs-, Wissenschafts-, Schul- und Universitätsgeschichte, Sozial- und Kulturgeschichte + 17. Jahrundert, 19. Jahrhundert, 20. Jahrhundert + 10.15463/rec.xxxxxxx + Deutsch + Slowenisch-deutscher Sprachvergleich, Riga / Handel, Tschechisch-deutscher Sprachkontakt, Arnim, Bettina von + http://recensio.net/r/8578c6f5e0304309991c1b12abeccc30 + francia-recensio/2010-4/mittelalter-moyen-age-500-1500/die-inszenierung-der-vergangenheit-im-mittelalter.pdf + http://www.recensio.net/rezensionen/zeitschriften/francia-recensio/2010-4/mittelalter-moyen-age-500-1500/die-inszenierung-der-vergangenheit-im-mittelalter + 2 + 3 + diff --git a/src/recensio/plone/browser/templates/export_rj_dara.pt b/src/recensio/plone/browser/templates/export_rj_dara.pt new file mode 100644 index 00000000..37c3ed24 --- /dev/null +++ b/src/recensio/plone/browser/templates/export_rj_dara.pt @@ -0,0 +1,115 @@ + + + Text + + + de + Rezension + + + en + Review + + + + ed891fc49fdbbb562278aa88e9ffb40d + 1 + + + + <language>de</language> + <titlename>Rezension: + <tal:title replace="title">Autor: Titel. Untertitel</tal:title></titlename> + + + + + + Hans + + Testperson + + + + + http://www.recensio.net/ + + 10.15463/rec.1517192636 + + 1970 + + + Download + + + + de + + Creative-Commons-Lizenz Namensnennung-Keine kommerzielle Nutzung-Keine Bearbeitung (CC-BY-NC-ND) + http://creativecommons.org/licenses/by-nc-nd/2.0/deed.de + + + + + + de + + Aufsatzsammlung + + + + + + 978-3-406-68822-5 + ISSN + Reviews + + + http://www.sehepunkte.de/2016/06/28546.html + URL + IsVariantFormOf + + + + + + Journal + + + + Herausgeber + + + + Titel der Zeitschrift + 1970 + + + + + ISSN + 978-3-406-68822-5 + ISSN + + + + + + diff --git a/src/recensio/plone/browser/templates/export_rm.pt b/src/recensio/plone/browser/templates/export_rm.pt new file mode 100644 index 00000000..4a0c40b0 --- /dev/null +++ b/src/recensio/plone/browser/templates/export_rm.pt @@ -0,0 +1,61 @@ + + + + + 978-5-91022-116-5 + Русский + 896 + Moskva + AIRO-XXI + Quellen und Darstellungen zur Zeitgeschichte + 2: Burnye proryvy, rasbitye nadeždy. Russkie i nemcy v mežvoennye gody + Rossija i Germanija v XX veke + 2 + 2010 + + + Slowenien, Polen, Lettland, Deutschland, Tschechien + Sprachgeschichte, Wirtschaftsgeschichte, Literaturgeschichte, Siedlungs-, Stadt- und Ortsgeschichte, Bildungs-, Wissenschafts-, Schul- und Universitätsgeschichte, Sozial- und Kulturgeschichte + 17. Jahrundert, 19. Jahrhundert, 20. Jahrhundert + 10.15463/rec.xxxxxxx + Deutsch + + Blažek + David + + Slowenisch-deutscher Sprachvergleich, Riga / Handel, Tschechisch-deutscher Sprachkontakt, Arnim, Bettina von + http://recensio.net/r/8578c6f5e0304309991c1b12abeccc30 + francia-recensio/2010-4/mittelalter-moyen-age-500-1500/die-inszenierung-der-vergangenheit-im-mittelalter.pdf + http://www.recensio.net/rezensionen/zeitschriften/francia-recensio/2010-4/mittelalter-moyen-age-500-1500/die-inszenierung-der-vergangenheit-im-mittelalter + 2 + 3 + diff --git a/src/recensio/plone/browser/templates/export_rm_dara.pt b/src/recensio/plone/browser/templates/export_rm_dara.pt new file mode 100644 index 00000000..3e67cab1 --- /dev/null +++ b/src/recensio/plone/browser/templates/export_rm_dara.pt @@ -0,0 +1,142 @@ + + + Text + + + de + Rezension + + + en + Review + + + + ed891fc49fdbbb562278aa88e9ffb40d + 1 + + + + <language>de</language> + <titlename>Rezension: + <tal:title replace="title">Autor: Titel. Untertitel</tal:title></titlename> + + + + + + Hans + + Testperson + + + + + http://www.recensio.net/ + + 10.15463/rec.1517192636 + + 1970 + + + Download + + + + de + + Creative-Commons-Lizenz Namensnennung-Keine kommerzielle Nutzung-Keine Bearbeitung (CC-BY-NC-ND) + http://creativecommons.org/licenses/by-nc-nd/2.0/deed.de + + + + + + de + + Aufsatzsammlung + + + + + + 978-3-406-68822-5 + ISBN + Reviews + + + http://www.sehepunkte.de/2016/06/28546.html + URL + IsVariantFormOf + + + + + + BookOrMonograph + + + It seems to happen quite often that there is an editor/author entry where both first + name and last name are the empty string. This causes an error on the dara server if + we don't filter it out. + + + + + Hans + + Testautor + + + + + + + + Testautor + , + Hans + + + + + + Titel der Monographie + 1970 + + + + + + + + ISBN + 978-3-406-68822-5 + ISBN + + + + + + diff --git a/src/recensio/plone/browser/templates/search.pt b/src/recensio/plone/browser/templates/search.pt index b05c8480..e818d169 100644 --- a/src/recensio/plone/browser/templates/search.pt +++ b/src/recensio/plone/browser/templates/search.pt @@ -1,74 +1,120 @@ - - - - - - - - - - - - - - - -
- -