diff --git a/Cerebrum/modules/greg/__init__.py b/Cerebrum/modules/greg/__init__.py index 19bee082e..175e1fa76 100644 --- a/Cerebrum/modules/greg/__init__.py +++ b/Cerebrum/modules/greg/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2021 University of Oslo, Norway +# Copyright 2021-2023 University of Oslo, Norway # # This file is part of Cerebrum. # @@ -43,7 +43,11 @@ Configuration ------------- -cereconf.CLASS_CONSTANTS +``cereconf.CLASS_CONSTANTS`` Must include ``Cerebrum.modules.greg.constants/GregConstants``, to provide Greg-related constants. + +``cereconf.GREG_IMPORT`` + Can be used to override the default + ``Cerebrum.modules.greg.importer/GregImporter`` class in scripts. """ diff --git a/Cerebrum/modules/greg/importer.py b/Cerebrum/modules/greg/importer.py index 52312002c..0000f194e 100644 --- a/Cerebrum/modules/greg/importer.py +++ b/Cerebrum/modules/greg/importer.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2021 University of Oslo, Norway +# Copyright 2021-2023 University of Oslo, Norway # # This file is part of Cerebrum. # @@ -17,7 +17,9 @@ # You should have received a copy of the GNU General Public License # along with Cerebrum; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. -""" Greg person import/update. """ +""" +Greg person import/update. +""" from __future__ import ( absolute_import, division, @@ -27,6 +29,8 @@ import logging import datetime +import cereconf + from Cerebrum.Utils import Factory from Cerebrum.modules.import_utils.matcher import ( OuMatcher, @@ -39,14 +43,30 @@ PersonNameSync, ) from Cerebrum.utils import date_compat +from Cerebrum.utils.module import resolve -from .consent import sync_greg_consent from .datasource import GregDatasource from .mapper import GregMapper logger = logging.getLogger(__name__) +def get_import_class(cereconf=cereconf): + """ + Get preferred import class from config module/object *cereconf*. + + TODO: Or should we re-factor the greg client config into a full greg + config, with import class and everything? + """ + import_spec = getattr(cereconf, 'GREG_IMPORT', None) + if import_spec: + cls = resolve(import_spec) + else: + cls = GregImporter + logger.info("greg import class=%s", repr(cls)) + return cls + + class GregImporter(object): REQUIRED_PERSON_ID = ( @@ -58,9 +78,9 @@ class GregImporter(object): 'GREG_PID', ) - CONSENT_GROUPS = { - 'greg-publish': sync_greg_consent, - } + # Map consent name to + # `Cerebrum.modules.import_utils.group.GroupMembershipSetter` + CONSENT_GROUPS = {} mapper = GregMapper() diff --git a/Cerebrum/modules/greg/tasks.py b/Cerebrum/modules/greg/tasks.py index 69f36aecf..a84690f84 100644 --- a/Cerebrum/modules/greg/tasks.py +++ b/Cerebrum/modules/greg/tasks.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2021 University of Oslo, Norway +# Copyright 2021-2023 University of Oslo, Norway # # This file is part of Cerebrum. # @@ -19,7 +19,6 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Tasks related to the Greg guest import. - """ import logging @@ -33,12 +32,32 @@ class GregImportTasks(queue_handler.QueueHandler): - """ This object defines the 'greg-guest' tasks queue. """ + """ This object defines the 'greg-person' task queues. """ queue = 'greg-person' manual_sub = 'manual' max_attempts = 20 + def __init__(self, client, import_class): + self._client = client + self._import_class = import_class + + def _callback(self, db, task): + greg_id = task.key + logger.info('Updating greg_id=%s', greg_id) + importer = self._import_class(db, client=self._client) + importer.handle_reference(greg_id) + logger.info('Updated greg_id=%s', greg_id) + + # TODO: Should we have the importer return potential new tasks? If so, + # we could rely on the default *handle_task* implementation for + # re-queueing. + # + # *Or* should the import itself add potential tasks to the queue? It + # kind of depends on whether we want the option to run the importer + # *without* adding new tasks to the queue... + return [] + @classmethod def create_manual_task(cls, reference, sub=manual_sub, nbf=None): """ Create a manual task. """ diff --git a/Cerebrum/modules/greg/consent.py b/Cerebrum/modules/import_utils/groups.py similarity index 79% rename from Cerebrum/modules/greg/consent.py rename to Cerebrum/modules/import_utils/groups.py index 1eefd5dee..06968cae0 100644 --- a/Cerebrum/modules/greg/consent.py +++ b/Cerebrum/modules/import_utils/groups.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2021 University of Oslo, Norway +# Copyright 2021-2023 University of Oslo, Norway # # This file is part of Cerebrum. # @@ -18,24 +18,13 @@ # along with Cerebrum; if not, write to the Free Software Foundation, # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ -Greg consent functionality. +Sync group membership for a given entity. """ import logging -from Cerebrum.group.template import GroupTemplate - logger = logging.getLogger(__name__) -GREG_CONSENT_GROUP = GroupTemplate( - group_name='greg-aktivt-samtykke', - group_description='Guests who consents to electronic publication', - group_type='internal-group', - group_visibility='A', -) - - -# TODO: Should this be a 'generic' import_utils class? class GroupMembershipSetter(object): """ Set membership for a single entity in a given group. @@ -60,9 +49,8 @@ def __init__(self, get_group): connection/transaction to use, and should return the Cerebrum.Group.Group object to update. - Would typically be a - py:class:`Cerebrum.group.template.GroupTemplate` or similar - callable object. + Would typically be a :class:`Cerebrum.group.template.GroupTemplate` + or similar callable object. """ self.get_group = get_group @@ -73,6 +61,15 @@ def __repr__(self): ) def __call__(self, db, entity_id, set_member): + """ + Ensure entity_id is or isn't a member of this group. + + :type db: Cerebrum.database.Database + :param int entity_id: member id to sync + :param bool set_member: if entity_id should be a member + + :returns bool: True if membership was changed + """ group = self.get_group(db) is_member = group.has_member(entity_id) @@ -80,10 +77,12 @@ def __call__(self, db, entity_id, set_member): logger.info('adding entity_id=%d to group %s (%d)', entity_id, group.group_name, group.entity_id) group.add_member(entity_id) - elif not set_member and is_member: + return True + + if not set_member and is_member: logger.info('removing entity_id=%d from group %s (%d)', entity_id, group.group_name, group.entity_id) group.remove_member(entity_id) + return True - -sync_greg_consent = GroupMembershipSetter(GREG_CONSENT_GROUP) + return False diff --git a/Cerebrum/modules/no/uio/greg_import.py b/Cerebrum/modules/no/uio/greg_import.py new file mode 100644 index 000000000..a68a8250f --- /dev/null +++ b/Cerebrum/modules/no/uio/greg_import.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2023 University of Oslo, Norway +# +# This file is part of Cerebrum. +# +# Cerebrum is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# Cerebrum is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Cerebrum; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. +""" +UiO-specific Greg import logic. +""" +from Cerebrum.group.template import GroupTemplate +from Cerebrum.modules.greg import mapper +from Cerebrum.modules.greg import importer +from Cerebrum.modules.import_utils.groups import GroupMembershipSetter + + +GREG_CONSENT_GROUP = GroupTemplate( + group_name='greg-aktivt-samtykke', + group_description='Guests who consents to electronic publication', + group_type='internal-group', + group_visibility='A', +) + +sync_greg_consent = GroupMembershipSetter(GREG_CONSENT_GROUP) + + +class UioGregMapper(mapper.GregMapper): + pass + + +class UioGregImporter(importer.GregImporter): + + CONSENT_GROUPS = { + 'greg-publish': sync_greg_consent, + } + + mapper = UioGregMapper() diff --git a/Cerebrum/modules/tasks/queue_handler.py b/Cerebrum/modules/tasks/queue_handler.py index 73aeea4db..e21f5f89e 100644 --- a/Cerebrum/modules/tasks/queue_handler.py +++ b/Cerebrum/modules/tasks/queue_handler.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright 2021 University of Oslo, Norway +# Copyright 2021-2023 University of Oslo, Norway # # This file is part of Cerebrum. # @@ -29,7 +29,10 @@ logger = logging.getLogger(__name__) -delay_on_error = backoff.Backoff( +# Default backoff for errors/retries in QueueHandler. This backoff yields +# time deltas 03:45, 07:30, 15:00, 30:00, 1:00:00, ... - before truncating at +# 12:00:00 after 10 attempts. This should be a good backoff for most tasks. +default_retry_delay = backoff.Backoff( backoff.Exponential(2), backoff.Factor(datetime.timedelta(hours=1) / 16), backoff.Truncate(datetime.timedelta(hours=12)), @@ -60,6 +63,9 @@ class QueueHandler(object): # when to give up on a task max_attempts = 20 + # next delay (timedelta) after *n* failed attempts + get_retry_delay = default_retry_delay + def __init__(self, callback): self._callback = callback @@ -67,9 +73,9 @@ def get_retry_task(self, task, error): """ Create a retry task from a failed task. """ retry = task_models.copy_task(task) retry.queue = self.queue - retry.sub = self.retry_sub or "" + retry.sub = self.retry_sub or task.sub retry.attempts = task.attempts + 1 - retry.nbf = now() + delay_on_error(task.attempts + 1) + retry.nbf = now() + self.get_retry_delay(task.attempts + 1) retry.reason = 'retry: failed_at={} error={}'.format(now(), error) return retry diff --git a/contrib/greg/greg-manual-import.py b/contrib/greg/greg-manual-import.py index 562c64d0f..858e71ea8 100644 --- a/contrib/greg/greg-manual-import.py +++ b/contrib/greg/greg-manual-import.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright 2021 University of Oslo, Norway +# Copyright 2021-2023 University of Oslo, Norway # # This file is part of Cerebrum. # @@ -31,7 +31,6 @@ unicode_literals, ) import argparse -import functools import logging import Cerebrum.logutils @@ -39,7 +38,7 @@ from Cerebrum.Utils import Factory from Cerebrum.database.ctx import db_context from Cerebrum.modules.greg.client import get_client -from Cerebrum.modules.greg.importer import GregImporter +from Cerebrum.modules.greg.importer import get_import_class from Cerebrum.utils.argutils import add_commit_args logger = logging.getLogger(__name__) @@ -81,13 +80,11 @@ def main(inargs=None): logger.info("start %s", parser.prog) logger.debug("args: %r", args) - # we don't really need the full TaskImportConfig here, but it's easier to - # re-use the existing config. client = get_client(args.config) - get_import = functools.partial(GregImporter, client=client) + import_class = get_import_class() with db_context(get_db(), not args.commit) as db: - greg_import = get_import(db) + greg_import = import_class(db, client=client) logger.info('handle reference=%r', args.reference) greg_import.handle_reference(args.reference) diff --git a/contrib/greg/greg-process-tasks.py b/contrib/greg/greg-process-tasks.py index 9a3887b7e..c0c5573a2 100644 --- a/contrib/greg/greg-process-tasks.py +++ b/contrib/greg/greg-process-tasks.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # -# Copyright 2021 University of Oslo, Norway +# Copyright 2021-2023 University of Oslo, Norway # # This file is part of Cerebrum. # @@ -20,14 +20,13 @@ # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. """ Process tasks on the greg import queues. """ import argparse -import functools import logging import Cerebrum.logutils import Cerebrum.logutils.options import Cerebrum.Errors from Cerebrum.modules.greg.client import get_client -from Cerebrum.modules.greg.importer import GregImporter +from Cerebrum.modules.greg.importer import get_import_class from Cerebrum.modules.greg.tasks import GregImportTasks from Cerebrum.modules.tasks.queue_processor import QueueProcessor from Cerebrum.utils.argutils import add_commit_args @@ -35,16 +34,6 @@ logger = logging.getLogger(__name__) -def task_callback(db, task, client): - """ Callback for the GregImportTasks queue handler. """ - greg_id = task.key - logger.info('Updating greg_id=%s', greg_id) - importer = GregImporter(db, client=client) - importer.handle_reference(greg_id) - logger.info('Updated greg_id=%s', greg_id) - return [] - - def main(inargs=None): parser = argparse.ArgumentParser( description='Process the greg-person import task queues', @@ -75,12 +64,11 @@ def main(inargs=None): dryrun = not args.commit client = get_client(args.config) - callback = functools.partial(task_callback, client=client) + import_class = get_import_class() + queue_handler = GregImportTasks(client=client, import_class=import_class) # The QueueProcessor gets db and does commit/rollback according to dryrun - proc = QueueProcessor(GregImportTasks(callback), - limit=args.limit, - dryrun=dryrun) + proc = QueueProcessor(queue_handler, limit=args.limit, dryrun=dryrun) tasks = proc.select_tasks() for task in tasks: