From a417a0d5b5d7c82bc3263ecd967f29c097db7455 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Wed, 9 Sep 2026 22:24:48 +0200 Subject: [PATCH 1/2] feat: DataSHIELD quotas API and command Add DataSHIELDQuotaService, wrapping the /service/r/quotas and /service/r/quota/{id} entry points with the DataSHIELD context: list, get, add, update and delete a quota, and read the usage of a user or of the current user. Expose it as the datashield-quota command, in both the Typer and the legacy consoles. --- obiba_opal/__init__.py | 2 + obiba_opal/commands.py | 101 ++++++++++ obiba_opal/console.py | 5 + obiba_opal/console_legacy.py | 8 + obiba_opal/datashield.py | 364 +++++++++++++++++++++++++++++++++++ tests/test_datashield.py | 89 +++++++++ 6 files changed, 569 insertions(+) create mode 100644 obiba_opal/datashield.py create mode 100644 tests/test_datashield.py diff --git a/obiba_opal/__init__.py b/obiba_opal/__init__.py index 0473366..c5ebd0a 100644 --- a/obiba_opal/__init__.py +++ b/obiba_opal/__init__.py @@ -68,6 +68,7 @@ ) from obiba_opal.sql import SQLService, SQLHistoryService from obiba_opal.security import EncryptService, DecryptService +from obiba_opal.datashield import DataSHIELDQuotaService __all__ = [ "UriBuilder", @@ -133,4 +134,5 @@ "SQLHistoryService", "EncryptService", "DecryptService", + "DataSHIELDQuotaService", ] diff --git a/obiba_opal/commands.py b/obiba_opal/commands.py index b4dd536..0e1c41c 100644 --- a/obiba_opal/commands.py +++ b/obiba_opal/commands.py @@ -74,6 +74,7 @@ ) from obiba_opal.sql import SQLService, SQLHistoryService from obiba_opal.security import EncryptService, DecryptService +from obiba_opal.datashield import DataSHIELDQuotaService def _make_args_with_globals( @@ -3418,3 +3419,103 @@ def analysis_plugin_command( json=json_output, ) AnalysisCommand.do_command(args) + + +# ============================================================================= +# DataSHIELD Commands +# ============================================================================= + + +def datashield_quota_command( + ctx: typer.Context, + opal: str = typer.Option("http://localhost:8080", "--opal", "-o", help="Opal server base url"), + user: str | None = typer.Option( + None, "--user", "-u", help="Credentials auth: user name (password will be requested if not provided)" + ), + password: str | None = typer.Option( + None, "--password", "-p", help="Credentials auth: user password (requires a user name)" + ), + token: str | None = typer.Option(None, "--token", "-tk", help="Token auth: User access token"), + ssl_cert: str | None = typer.Option( + None, "--ssl-cert", "-sc", help="Two-way SSL auth: certificate/public key file (requires a private key)" + ), + ssl_key: str | None = typer.Option( + None, "--ssl-key", "-sk", help="Two-way SSL auth: private key file (requires a certificate)" + ), + verbose: bool = typer.Option(False, "--verbose", "-v", help="Verbose output"), + no_ssl_verify: bool = typer.Option( + False, "--no-ssl-verify", "-nv", help="Do not verify SSL certificates for HTTPS." + ), + id: str | None = typer.Option( + None, + "--id", + "-id", + help="Quota identifier. Not specifying the quota identifier, will get the list of the DataSHIELD quotas.", + ), + type: str | None = typer.Option( + None, "--type", "-ty", help="Subject type: system, group, user. Default is system." + ), + subject: str | None = typer.Option( + None, + "--subject", + "-s", + help="Subject name: the user or the group name (required when subject type is user or group).", + ), + metric: str | None = typer.Option( + None, "--metric", "-m", help="Usage metric: execution-time, session-time. Default is execution-time." + ), + period: str | None = typer.Option( + None, "--period", "-pd", help="Rolling period the usage is summed over: daily, weekly. Default is weekly." + ), + limit: float | None = typer.Option( + None, "--limit", "-lm", help="Usage allowance, in minutes (zero forbids DataSHIELD for this subject)." + ), + disabled: bool = typer.Option( + False, "--disabled", "-di", help="Disable the quota (on add, if omitted the quota is enabled by default)." + ), + enabled: bool = typer.Option( + False, + "--enabled", + "-en", + help="Enable the quota (on update, when neither --enabled nor --disabled is specified, the quota is left " + "as it is).", + ), + fetch: bool = typer.Option(False, "--fetch", "-fe", help="Fetch one or multiple quota(s)."), + add: bool = typer.Option(False, "--add", "-a", help="Add a quota."), + update: bool = typer.Option(False, "--update", "-ud", help="Update a quota (requires a quota identifier)."), + delete: bool = typer.Option(False, "--delete", "-de", help="Delete a quota (requires a quota identifier)."), + usage: bool = typer.Option( + False, + "--usage", + "-us", + help="Get the quota usage of the user specified by --subject, or of the current user when omitted.", + ), + json_output: bool = typer.Option(False, "--json", "-j", help="Pretty JSON formatting of the response"), +): + """Manage DataSHIELD usage quotas.""" + args = _make_args_with_globals( + ctx, + opal=opal, + user=user, + password=password, + token=token, + ssl_cert=ssl_cert, + ssl_key=ssl_key, + verbose=verbose, + no_ssl_verify=no_ssl_verify, + id=id, + type=type, + subject=subject, + metric=metric, + period=period, + limit=limit, + disabled=disabled, + enabled=enabled, + fetch=fetch, + add=add, + update=update, + delete=delete, + usage=usage, + json=json_output, + ) + DataSHIELDQuotaService.do_command(args) diff --git a/obiba_opal/console.py b/obiba_opal/console.py index 2c132c0..9cfa402 100755 --- a/obiba_opal/console.py +++ b/obiba_opal/console.py @@ -200,6 +200,11 @@ def wrapper(*args, **kwargs) -> None: help="Analyses a project variables using external R plugins.", )(handle_exceptions(cmd.analysis_plugin_command)) +# DataSHIELD commands +app.command(name="datashield-quota", help="Manage DataSHIELD usage quotas.")( + handle_exceptions(cmd.datashield_quota_command) +) + # ============================================================================= # Global options callback diff --git a/obiba_opal/console_legacy.py b/obiba_opal/console_legacy.py index 544646b..9a596cd 100644 --- a/obiba_opal/console_legacy.py +++ b/obiba_opal/console_legacy.py @@ -72,6 +72,7 @@ ) from obiba_opal.sql import SQLService, SQLHistoryService from obiba_opal.security import EncryptService, DecryptService +from obiba_opal.datashield import DataSHIELDQuotaService def prompt_password(): @@ -546,6 +547,13 @@ def run(): SQLHistoryService.add_arguments, SQLHistoryService.do_command, ) + add_subcommand( + subparsers, + "datashield-quota", + "Manage DataSHIELD usage quotas.", + DataSHIELDQuotaService.add_arguments, + DataSHIELDQuotaService.do_command, + ) # Execute selected command args = parser.parse_args() diff --git a/obiba_opal/datashield.py b/obiba_opal/datashield.py new file mode 100644 index 0000000..71a9a07 --- /dev/null +++ b/obiba_opal/datashield.py @@ -0,0 +1,364 @@ +""" +Opal DataSHIELD management. +""" + +import obiba_opal.core as core +import json + + +class DataSHIELDQuotaService: + """ + DataSHIELD usage quotas management service. + + A quota is an allowance of R usage, in the DataSHIELD execution context, for a subject (the system default, + a group or a user) and a usage metric. The usage consumed against it is summed over the rolling window + of the quota period. + """ + + CONTEXT = "DataSHIELD" + + SUBJECT_TYPES = ("SYSTEM", "GROUP", "USER") + METRICS = ("EXECUTION_TIME", "SESSION_TIME") + PERIODS = ("DAILY", "WEEKLY") + + def __init__(self, client: core.OpalClient, verbose: bool = False): + self.client = client + self.verbose = verbose + + @classmethod + def add_arguments(cls, parser): + """ + Add command specific options + """ + parser.add_argument( + "--id", + "-id", + required=False, + help="Quota identifier. Not specifying the quota identifier, will get the list of the DataSHIELD quotas.", + ) + parser.add_argument( + "--type", + "-ty", + required=False, + help=f"Subject type: {', '.join(cls.SUBJECT_TYPES).lower()}. Default is system.", + ) + parser.add_argument( + "--subject", + "-s", + required=False, + help="Subject name: the user or the group name (required when subject type is user or group).", + ) + parser.add_argument( + "--metric", + "-m", + required=False, + help=f"Usage metric: {', '.join(cls.METRICS).lower().replace('_', '-')}. Default is execution-time.", + ) + parser.add_argument( + "--period", + "-pd", + required=False, + help=f"Rolling period the usage is summed over: {', '.join(cls.PERIODS).lower()}. Default is weekly.", + ) + parser.add_argument( + "--limit", + "-lm", + required=False, + help="Usage allowance, in minutes (zero forbids DataSHIELD for this subject).", + ) + parser.add_argument( + "--disabled", + "-di", + action="store_true", + required=False, + help="Disable the quota (on add, if omitted the quota is enabled by default).", + ) + parser.add_argument( + "--enabled", + "-en", + action="store_true", + required=False, + help="Enable the quota (on update, when neither --enabled nor --disabled is specified, the quota is " + "left as it is).", + ) + parser.add_argument( + "--fetch", + "-fe", + action="store_true", + required=False, + help="Fetch one or multiple quota(s).", + ) + parser.add_argument("--add", "-a", action="store_true", help="Add a quota.") + parser.add_argument( + "--update", + "-ud", + action="store_true", + required=False, + help="Update a quota (requires a quota identifier).", + ) + parser.add_argument( + "--delete", + "-de", + action="store_true", + required=False, + help="Delete a quota (requires a quota identifier).", + ) + parser.add_argument( + "--usage", + "-us", + action="store_true", + required=False, + help="Get the quota usage of the user specified by --subject, or of the current user when omitted.", + ) + parser.add_argument( + "--json", + "-j", + action="store_true", + help="Pretty JSON formatting of the response", + ) + + @classmethod + def do_command(cls, args): + """ + Execute quota command + """ + # Build and send request + client = core.OpalClient.build(core.OpalClient.LoginInfo.parse(args)) + try: + service = DataSHIELDQuotaService(client, args.verbose) + limit_millis = cls._minutes_to_millis(args.limit) + enabled = cls._parse_enabled(args) + if args.add: + res = service.add_quota( + limit_millis, + args.type, + args.subject, + args.metric, + args.period, + True if enabled is None else enabled, + ) + core.Formatter.print_json(res, args.json) + elif args.update: + res = service.update_quota( + args.id, + limit_millis, + args.type, + args.subject, + args.metric, + args.period, + enabled, + ) + core.Formatter.print_json(res, args.json) + elif args.delete: + service.delete_quota(args.id) + elif args.usage: + res = service.get_usage(args.subject) if args.subject else service.get_current_usage() + core.Formatter.print_json(res, args.json) + else: + res = service.get_quota(args.id) if args.id else service.get_quotas() + core.Formatter.print_json(res, args.json) + finally: + client.close() + + def get_quotas(self) -> list: + """ + Get the DataSHIELD quotas. + """ + request = self._make_request() + response = request.get().resource(self._make_ws()).send() + return response.from_json() + + def get_quota(self, id) -> dict: + """ + Get a DataSHIELD quota. + + :param id: The quota identifier + """ + if not id: + raise ValueError("The identifier of the quota to fetch is required") + request = self._make_request() + response = request.get().resource(self._make_ws(id)).send() + return response.from_json() + + def add_quota( + self, + limit_millis: int, + subject_type: str = "SYSTEM", + principal: str = None, + metric: str = "EXECUTION_TIME", + period: str = "WEEKLY", + enabled: bool = True, + ) -> dict: + """ + Add a DataSHIELD quota. A subject that already has a quota for this metric has it replaced. + + :param limit_millis: The usage allowance, in milliseconds of whatever the metric names (zero forbids + DataSHIELD for this subject) + :param subject_type: The subject type: 'SYSTEM' (default), 'GROUP' or 'USER' + :param principal: The user name or the group name (required when the subject type is 'GROUP' or 'USER') + :param metric: The usage metric: 'EXECUTION_TIME' (default) or 'SESSION_TIME' + :param period: The rolling period the usage is summed over: 'WEEKLY' (default) or 'DAILY' + :param enabled: Whether the quota applies (True is default) + """ + quota = self._make_quota(limit_millis, subject_type, principal, metric, period, enabled) + request = self._make_request() + request.accept_json().content_type_json() + uri = core.UriBuilder(["service", "r", "quotas"]).build() + response = request.post().resource(uri).content(json.dumps(quota)).send() + return response.from_json() + + def update_quota( + self, + id, + limit_millis: int = None, + subject_type: str = None, + principal: str = None, + metric: str = None, + period: str = None, + enabled: bool = None, + ) -> dict: + """ + Update a DataSHIELD quota: the values that are not provided are left unchanged. + + :param id: The quota identifier + :param limit_millis: The usage allowance, in milliseconds of whatever the metric names + :param subject_type: The subject type: 'SYSTEM', 'GROUP' or 'USER' + :param principal: The user name or the group name (required when the subject type is 'GROUP' or 'USER') + :param metric: The usage metric: 'EXECUTION_TIME' or 'SESSION_TIME' + :param period: The rolling period the usage is summed over: 'DAILY' or 'WEEKLY' + :param enabled: Whether the quota applies + """ + if not id: + raise ValueError("The identifier of the quota to update is required") + current = self.get_quota(id) + quota = self._make_quota( + current["limitMillis"] if limit_millis is None else limit_millis, + current["subjectType"] if subject_type is None else subject_type, + current["principal"] if principal is None else principal, + current["metric"] if metric is None else metric, + current["period"] if period is None else period, + current["enabled"] if enabled is None else enabled, + ) + quota["id"] = current["id"] + request = self._make_request() + request.accept_json().content_type_json() + response = request.put().resource(self._make_ws(id)).content(json.dumps(quota)).send() + return response.from_json() + + def delete_quota(self, id): + """ + Delete a DataSHIELD quota. The subject becomes unlimited again, unless a broader quota still applies. + + :param id: The quota identifier + """ + if not id: + raise ValueError("The identifier of the quota to delete is required") + request = self._make_request() + request.delete().resource(self._make_ws(id)).send() + + def get_usage(self, user: str) -> list: + """ + Get, for each usage metric, the DataSHIELD quota that applies to a user and what they have consumed + against it. Reading the usage of another user requires administration permissions. + + :param user: The user name + """ + if not user: + raise ValueError("The name of the user to get the quota usage of is required") + request = self._make_request() + uri = core.UriBuilder(["service", "r", "quotas", "_usage"]).query("context", self.CONTEXT) + response = request.get().resource(uri.query("user", user).build()).send() + return response.from_json() + + def get_current_usage(self) -> list: + """ + Get, for each usage metric, the DataSHIELD quota that applies to the current user and what they have + consumed against it. + """ + request = self._make_request() + uri = core.UriBuilder(["service", "r", "quotas", "_current"]).query("context", self.CONTEXT).build() + response = request.get().resource(uri).send() + return response.from_json() + + def _make_quota( + self, + limit_millis: int, + subject_type: str, + principal: str, + metric: str, + period: str, + enabled: bool, + ) -> dict: + """ + Make the quota representation, validating and normalizing the enumerated values + """ + if limit_millis is None: + raise ValueError("The quota limit is required") + limit_millis = int(limit_millis) + if limit_millis < 0: + raise ValueError("The quota limit cannot be negative") + subject_type = self._normalize(subject_type, "SYSTEM", self.SUBJECT_TYPES, "subject type") + metric = self._normalize(metric, "EXECUTION_TIME", self.METRICS, "usage metric") + period = self._normalize(period, "WEEKLY", self.PERIODS, "period") + if subject_type == "SYSTEM": + principal = "" + elif not principal: + raise ValueError("The subject name is required") + return { + "context": self.CONTEXT, + "subjectType": subject_type, + "principal": principal, + "metric": metric, + "period": period, + "limitMillis": limit_millis, + "enabled": enabled, + } + + @classmethod + def _normalize(cls, value: str, default: str, allowed: tuple, label: str) -> str: + """ + Normalize an enumerated value, dashes being accepted in place of underscores + """ + if not value: + return default + normalized = value.strip().upper().replace("-", "_") + if normalized not in allowed: + valid = ", ".join(allowed).lower().replace("_", "-") + raise ValueError(f"Valid values for the {label} are: {valid}") + return normalized + + @classmethod + def _parse_enabled(cls, args) -> bool: + """ + Get the enabled state to apply: None when it was not specified, so that an update leaves it as it is + """ + if args.enabled and args.disabled: + raise ValueError("The quota cannot be both enabled and disabled") + if args.disabled: + return False + return True if args.enabled else None + + @classmethod + def _minutes_to_millis(cls, minutes) -> int: + """ + Convert a limit expressed in minutes into milliseconds + """ + if minutes is None or minutes == "": + return None + return int(round(float(minutes) * 60000)) + + def _make_ws(self, id=None): + """ + Build the web service resource path + """ + if id: + return core.UriBuilder(["service", "r", "quota", str(id)]).build() + return core.UriBuilder(["service", "r", "quotas"]).query("context", self.CONTEXT).build() + + def _make_request(self, fail_safe: bool = False): + request = self.client.new_request() + if not fail_safe: + request.fail_on_error() + if self.verbose: + request.verbose() + return request diff --git a/tests/test_datashield.py b/tests/test_datashield.py new file mode 100644 index 0000000..3479ab6 --- /dev/null +++ b/tests/test_datashield.py @@ -0,0 +1,89 @@ +import pytest +from obiba_opal import DataSHIELDQuotaService, HTTPError +from tests.utils import make_client + + +class TestClass: + @classmethod + def setup_class(cls): + cls.client = make_client() + + @classmethod + def teardown_class(cls): + cls.client.close() + + def _service(self): + """ + The quotas API is only available from Opal 6, skip when the test server does not have it + """ + service = DataSHIELDQuotaService(self.client) + try: + service.get_quotas() + except HTTPError as e: + if e.code == 404: + pytest.skip("The Opal server does not support DataSHIELD quotas") + raise + return service + + @pytest.mark.integration + def test_quotas(self): + service = self._service() + quotas = service.get_quotas() + assert isinstance(quotas, list) + for quota in quotas: + assert quota["context"] == "DataSHIELD" + + @pytest.mark.integration + def test_add_update_delete_quota(self): + service = self._service() + quota = service.add_quota(60 * 60000, "user", "dsuser", "session-time", "daily") + try: + assert quota["context"] == "DataSHIELD" + assert quota["subjectType"] == "USER" + assert quota["principal"] == "dsuser" + assert quota["metric"] == "SESSION_TIME" + assert quota["period"] == "DAILY" + assert quota["limitMillis"] == 3600000 + assert quota["enabled"] + + fetched = service.get_quota(quota["id"]) + assert fetched["id"] == quota["id"] + + updated = service.update_quota(quota["id"], limit_millis=120 * 60000, enabled=False) + assert updated["limitMillis"] == 7200000 + assert not updated["enabled"] + # the values that were not provided are left unchanged + assert updated["principal"] == "dsuser" + assert updated["period"] == "DAILY" + finally: + service.delete_quota(quota["id"]) + + @pytest.mark.integration + def test_current_usage(self): + service = self._service() + usages = service.get_current_usage() + assert isinstance(usages, list) + for usage in usages: + assert usage["context"] == "DataSHIELD" + assert "metric" in usage + assert "usedMillis" in usage + + def test_invalid_quota(self): + service = DataSHIELDQuotaService(self.client) + with pytest.raises(ValueError): + # a group quota requires a subject name + service.add_quota(60000, "group") + with pytest.raises(ValueError): + service.add_quota(60000, "nosuchtype", "dsuser") + with pytest.raises(ValueError): + service.add_quota(60000, "user", "dsuser", "nosuchmetric") + with pytest.raises(ValueError): + service.add_quota(60000, "user", "dsuser", "session-time", "nosuchperiod") + with pytest.raises(ValueError): + service.add_quota(-1) + with pytest.raises(ValueError): + service.get_quota(None) + with pytest.raises(ValueError): + service.delete_quota(None) + with pytest.raises(ValueError): + service.get_usage(None) From 5a6ec223adbe9fd13d28292c196c3dbe058aa2d6 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Wed, 9 Sep 2026 22:50:40 +0200 Subject: [PATCH 2/2] fix: address review comments on the DataSHIELD quota command Move the quota validation test out of the test class: the class setup builds a real client, which does network I/O, so an unmarked test in it made a "not integration" run depend on the test server. Validation happens before any request is sent, so the test needs no client at all. State in the help of --fetch that fetching is the default action. --- obiba_opal/commands.py | 4 +++- obiba_opal/datashield.py | 3 ++- tests/test_datashield.py | 42 ++++++++++++++++++++++------------------ 3 files changed, 28 insertions(+), 21 deletions(-) diff --git a/obiba_opal/commands.py b/obiba_opal/commands.py index 0e1c41c..13847ac 100644 --- a/obiba_opal/commands.py +++ b/obiba_opal/commands.py @@ -3480,7 +3480,9 @@ def datashield_quota_command( help="Enable the quota (on update, when neither --enabled nor --disabled is specified, the quota is left " "as it is).", ), - fetch: bool = typer.Option(False, "--fetch", "-fe", help="Fetch one or multiple quota(s)."), + fetch: bool = typer.Option( + False, "--fetch", "-fe", help="Fetch one or multiple quota(s). This is the default action." + ), add: bool = typer.Option(False, "--add", "-a", help="Add a quota."), update: bool = typer.Option(False, "--update", "-ud", help="Update a quota (requires a quota identifier)."), delete: bool = typer.Option(False, "--delete", "-de", help="Delete a quota (requires a quota identifier)."), diff --git a/obiba_opal/datashield.py b/obiba_opal/datashield.py index 71a9a07..51ea8ad 100644 --- a/obiba_opal/datashield.py +++ b/obiba_opal/datashield.py @@ -86,7 +86,7 @@ def add_arguments(cls, parser): "-fe", action="store_true", required=False, - help="Fetch one or multiple quota(s).", + help="Fetch one or multiple quota(s). This is the default action.", ) parser.add_argument("--add", "-a", action="store_true", help="Add a quota.") parser.add_argument( @@ -155,6 +155,7 @@ def do_command(cls, args): res = service.get_usage(args.subject) if args.subject else service.get_current_usage() core.Formatter.print_json(res, args.json) else: + # fetching is the default action, --fetch being accepted to state it explicitly res = service.get_quota(args.id) if args.id else service.get_quotas() core.Formatter.print_json(res, args.json) finally: diff --git a/tests/test_datashield.py b/tests/test_datashield.py index 3479ab6..a8299de 100644 --- a/tests/test_datashield.py +++ b/tests/test_datashield.py @@ -68,22 +68,26 @@ def test_current_usage(self): assert "metric" in usage assert "usedMillis" in usage - def test_invalid_quota(self): - service = DataSHIELDQuotaService(self.client) - with pytest.raises(ValueError): - # a group quota requires a subject name - service.add_quota(60000, "group") - with pytest.raises(ValueError): - service.add_quota(60000, "nosuchtype", "dsuser") - with pytest.raises(ValueError): - service.add_quota(60000, "user", "dsuser", "nosuchmetric") - with pytest.raises(ValueError): - service.add_quota(60000, "user", "dsuser", "session-time", "nosuchperiod") - with pytest.raises(ValueError): - service.add_quota(-1) - with pytest.raises(ValueError): - service.get_quota(None) - with pytest.raises(ValueError): - service.delete_quota(None) - with pytest.raises(ValueError): - service.get_usage(None) + +def test_invalid_quota(): + """ + The quota values are validated before any request is sent, so no server is needed here + """ + service = DataSHIELDQuotaService(None) + with pytest.raises(ValueError): + # a group quota requires a subject name + service.add_quota(60000, "group") + with pytest.raises(ValueError): + service.add_quota(60000, "nosuchtype", "dsuser") + with pytest.raises(ValueError): + service.add_quota(60000, "user", "dsuser", "nosuchmetric") + with pytest.raises(ValueError): + service.add_quota(60000, "user", "dsuser", "session-time", "nosuchperiod") + with pytest.raises(ValueError): + service.add_quota(-1) + with pytest.raises(ValueError): + service.get_quota(None) + with pytest.raises(ValueError): + service.delete_quota(None) + with pytest.raises(ValueError): + service.get_usage(None)