diff --git a/src/dstack/_internal/cli/services/configurators/run.py b/src/dstack/_internal/cli/services/configurators/run.py index d3273ba9c6..7429737a9a 100644 --- a/src/dstack/_internal/cli/services/configurators/run.py +++ b/src/dstack/_internal/cli/services/configurators/run.py @@ -894,7 +894,12 @@ def print_finished_message(run: Run): if ( termination_reason # A run that never started has no runner logs to read. - and termination_reason != JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY.value + and termination_reason + not in ( + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY.value, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_QUOTA.value, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_BALANCE.value, + ) ): console.print(f"Check [code]dstack logs -d {run.name}[/code] for more details.") diff --git a/src/dstack/_internal/cli/utils/fleet.py b/src/dstack/_internal/cli/utils/fleet.py index 811db7db9f..666ebf2f19 100644 --- a/src/dstack/_internal/cli/utils/fleet.py +++ b/src/dstack/_internal/cli/utils/fleet.py @@ -8,10 +8,15 @@ format_backend, format_entity_reference, ) +from dstack._internal.core.consts import NO_BALANCE_MESSAGE, NO_QUOTA_MESSAGE from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.fleets import Fleet, FleetNodesSpec, FleetStatus from dstack._internal.core.models.health import HealthStatus -from dstack._internal.core.models.instances import Instance, InstanceStatus +from dstack._internal.core.models.instances import ( + Instance, + InstanceStatus, + InstanceTerminationReason, +) from dstack._internal.core.models.resources import GPUSpec, ResourcesSpec from dstack._internal.utils.common import DateFormatter, pretty_date @@ -166,6 +171,12 @@ def get_fleets_table( if instance.status == InstanceStatus.TERMINATED and instance.termination_reason: instance_row["ERROR"] = instance.termination_reason + if ( + instance.termination_reason == InstanceTerminationReason.NO_OFFERS + and instance.termination_reason_message + in (NO_QUOTA_MESSAGE, NO_BALANCE_MESSAGE) + ): + instance_row["ERROR"] = instance.termination_reason_message.lower() # No row-level dimming: only the columns styled above are dimmed on instance rows add_row_from_dict(table, instance_row) diff --git a/src/dstack/_internal/core/consts.py b/src/dstack/_internal/core/consts.py index 9b60381bdc..65561d78bf 100644 --- a/src/dstack/_internal/core/consts.py +++ b/src/dstack/_internal/core/consts.py @@ -12,5 +12,9 @@ DSTACK_PROJECT_ENV = "DSTACK_PROJECT" DSTACK_SERVER_URL_ENV = "DSTACK_SERVER_URL" DSTACK_TOKEN_ENV = "DSTACK_TOKEN" +# Keep these messages stable while existing termination reasons carry quota/balance failures +# during rolling upgrades. Status formatting recognizes their exact values. +NO_QUOTA_MESSAGE = "No quota" +NO_BALANCE_MESSAGE = "No balance" # legacy AWS, Azure, GCP, and OCI image for older GPUs DSTACK_OS_IMAGE_WITH_PROPRIETARY_NVIDIA_KERNEL_MODULES = "0.10" diff --git a/src/dstack/_internal/core/models/instances.py b/src/dstack/_internal/core/models/instances.py index 68e5c11028..bd14defdd7 100644 --- a/src/dstack/_internal/core/models/instances.py +++ b/src/dstack/_internal/core/models/instances.py @@ -246,6 +246,8 @@ class InstanceTerminationReason(str, Enum): JOB_FINISHED = "job_finished" UNREACHABLE = "unreachable" NO_OFFERS = "no_offers" + # Recognize new reasons before storing them to support rolling upgrades. + NO_QUOTA = "no_quota" MASTER_FAILED = "master_failed" MAX_INSTANCES_LIMIT = "max_instances_limit" FLEET_SPEC_MISMATCH = "fleet_spec_mismatch" diff --git a/src/dstack/_internal/core/models/runs.py b/src/dstack/_internal/core/models/runs.py index a120390079..f7525330a2 100644 --- a/src/dstack/_internal/core/models/runs.py +++ b/src/dstack/_internal/core/models/runs.py @@ -138,6 +138,9 @@ def to_error(self) -> Optional[str]: class JobTerminationReason(str, Enum): # Set by the server FAILED_TO_START_DUE_TO_NO_CAPACITY = "failed_to_start_due_to_no_capacity" + # Recognize new reasons before storing them to support rolling upgrades. + FAILED_TO_START_DUE_TO_NO_QUOTA = "failed_to_start_due_to_no_quota" + FAILED_TO_START_DUE_TO_NO_BALANCE = "failed_to_start_due_to_no_balance" INTERRUPTED_BY_NO_CAPACITY = "interrupted_by_no_capacity" INSTANCE_UNREACHABLE = "instance_unreachable" INSTANCE_ACCESS_REVOKED = "instance_access_revoked" @@ -163,6 +166,8 @@ class JobTerminationReason(str, Enum): def to_status(self) -> JobStatus: mapping = { self.FAILED_TO_START_DUE_TO_NO_CAPACITY: JobStatus.FAILED, + self.FAILED_TO_START_DUE_TO_NO_QUOTA: JobStatus.FAILED, + self.FAILED_TO_START_DUE_TO_NO_BALANCE: JobStatus.FAILED, self.INTERRUPTED_BY_NO_CAPACITY: JobStatus.FAILED, self.INSTANCE_UNREACHABLE: JobStatus.FAILED, self.INSTANCE_ACCESS_REVOKED: JobStatus.FAILED, @@ -194,6 +199,9 @@ def to_retry_event(self) -> Optional[RetryEvent]: """ mapping = { self.FAILED_TO_START_DUE_TO_NO_CAPACITY: RetryEvent.NO_CAPACITY, + # Preserve existing no-capacity retry configurations for blocked offers. + self.FAILED_TO_START_DUE_TO_NO_QUOTA: RetryEvent.NO_CAPACITY, + self.FAILED_TO_START_DUE_TO_NO_BALANCE: RetryEvent.NO_CAPACITY, self.INTERRUPTED_BY_NO_CAPACITY: RetryEvent.INTERRUPTION, } default = RetryEvent.ERROR if self.to_status() == JobStatus.FAILED else None diff --git a/src/dstack/_internal/server/background/pipeline_tasks/fleets.py b/src/dstack/_internal/server/background/pipeline_tasks/fleets.py index 4904e7e3a2..57d2896789 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/fleets.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/fleets.py @@ -841,7 +841,12 @@ def _set_fail_instances_on_master_bootstrap_failure( if ( current_master_instance_model.status != InstanceStatus.TERMINATED - or current_master_instance_model.termination_reason != InstanceTerminationReason.NO_OFFERS + or current_master_instance_model.termination_reason + not in ( + InstanceTerminationReason.NO_OFFERS, + InstanceTerminationReason.NO_QUOTA, + InstanceTerminationReason.NO_BALANCE, + ) ): return diff --git a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py index bd869b8325..cead273c5e 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/instances/cloud_provisioning.py @@ -28,7 +28,8 @@ InstanceTerminationReason, ) from dstack._internal.core.models.placement import PlacementGroupConfiguration, PlacementStrategy -from dstack._internal.core.models.runs import JobProvisioningData +from dstack._internal.core.models.profiles import Profile +from dstack._internal.core.models.runs import JobProvisioningData, Requirements from dstack._internal.server import settings as server_settings from dstack._internal.server.background.pipeline_tasks.base import NOW_PLACEHOLDER from dstack._internal.server.background.pipeline_tasks.instances.common import ( @@ -45,7 +46,10 @@ get_instance_requirements, ) from dstack._internal.server.services.logging import fmt -from dstack._internal.server.services.offers import get_instance_offer_with_restricted_az +from dstack._internal.server.services.offers import ( + get_instance_offer_with_restricted_az, + get_offers_termination_reason_message, +) from dstack._internal.server.services.placement import ( can_use_placement_groups, get_fleet_placement_group_models, @@ -236,16 +240,72 @@ async def create_cloud_instance(instance_model: InstanceModel) -> ProcessResult: result.schedule_pg_deletion_except_id = placement_group_model.id return result + termination_reason_message = "All offers failed" if offers else "No offers found" + if not offers: + termination_reason_message = ( + await _get_no_offers_termination_reason_message( + instance_model=instance_model, + profile=profile, + requirements=requirements, + placement_group_model=placement_group_model, + master_job_provisioning_data=master_job_provisioning_data, + ) + or termination_reason_message + ) + set_status_update( update_map=result.instance_update_map, instance_model=instance_model, new_status=InstanceStatus.TERMINATED, termination_reason=InstanceTerminationReason.NO_OFFERS, - termination_reason_message="All offers failed" if offers else "No offers found", + termination_reason_message=termination_reason_message, ) return result +async def _get_no_offers_termination_reason_message( + instance_model: InstanceModel, + profile: Profile, + requirements: Requirements, + placement_group_model: Optional[PlacementGroupModel], + master_job_provisioning_data: Optional[JobProvisioningData], +) -> Optional[str]: + """Explain why no offers are available by refetching with unavailable offers included. + + Return "No quota" or "No balance" only if all matching offers have that reason. + Return None otherwise or if the lookup fails. + + Adds one offer lookup only when no available offers were found. Backend loading and + quota/balance checks may repeat DB reads; provider cache misses may repeat + provider calls. + """ + # TODO: Return InstanceTerminationReason values and persist NO_QUOTA/NO_BALANCE + # once all supported rolling-upgrade source versions recognize them. Until then, + # use messages so older replicas can read the shared DB. + # Refetching will still be needed: the initial lookup excludes unavailable offers. + try: + offers = await get_fleet_offers( + project=instance_model.project, + profile=profile, + requirements=requirements, + exclude_not_available=False, + fleet_model=instance_model.fleet, + placement_group=placement_group_model_to_placement_group_optional( + placement_group_model + ), + blocks="auto" if instance_model.total_blocks is None else instance_model.total_blocks, + master_job_provisioning_data=master_job_provisioning_data, + infer_master_job_provisioning_data_from_fleet_instances=False, + include_only_create_instance_supported_backends=True, + ) + return await run_async(get_offers_termination_reason_message, offers) + except Exception: + logger.warning( + "%s: failed to check unavailable offers", fmt(instance_model), exc_info=True + ) + return None + + async def _get_cluster_master_context( instance_model: InstanceModel, ) -> Optional[_ClusterMasterContext]: diff --git a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py index 1ab7875d04..2d93fb0d4f 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py @@ -119,6 +119,7 @@ from dstack._internal.server.services.offers import ( get_instance_offer_with_restricted_az, get_offers_by_requirements, + get_offers_termination_reason_message, ) from dstack._internal.server.services.pipelines import PipelineHinterProtocol from dstack._internal.server.services.placement import ( @@ -485,7 +486,7 @@ class _NewCapacityAttempts: class _FailedNewCapacityProvisioning: placement_group_cleanup: Optional[_PlacementGroupCleanup] message: Optional[str] = None - """Why the job could not be provisioned. `None` if the offers were never tried.""" + """Why the job could not be provisioned.""" @dataclass @@ -2569,18 +2570,34 @@ async def _provision_new_capacity( offer.region, ) continue - return _FailedNewCapacityProvisioning( - message=_get_new_capacity_failure_message( - fleet_name=fleet_model.name, - attempts=_NewCapacityAttempts( - total=len(offers), - tried=offers_tried, - skip_reasons=skip_reasons, - errors=offer_errors, - # Offers are left only if the attempt limit, not the offer list, ended the loop. - limit_reached=offers_taken < len(offers), - ), + termination_reason_message = _get_new_capacity_failure_message( + fleet_name=fleet_model.name, + attempts=_NewCapacityAttempts( + total=len(offers), + tried=offers_tried, + skip_reasons=skip_reasons, + errors=offer_errors, + # Offers are left only if the attempt limit, not the offer list, ended the loop. + limit_reached=offers_taken < len(offers), ), + ) + if not offers: + termination_reason_message = ( + await _get_no_offers_termination_reason_message( + project=project, + job_model=job_model, + run=run, + job=job, + profile=profile, + requirements=requirements, + volumes=volumes, + placement_group_model=placement_group_model, + master_job_provisioning_data=master_job_provisioning_data, + ) + or termination_reason_message + ) + return _FailedNewCapacityProvisioning( + message=termination_reason_message, placement_group_cleanup=_build_placement_group_cleanup( fleet_model=fleet_model, offers_tried=offers_tried, @@ -2590,6 +2607,52 @@ async def _provision_new_capacity( ) +async def _get_no_offers_termination_reason_message( + project: ProjectModel, + job_model: JobModel, + run: Run, + job: Job, + profile: Profile, + requirements: Requirements, + volumes: list[list[Volume]], + placement_group_model: Optional[PlacementGroupModel], + master_job_provisioning_data: Optional[JobProvisioningData], +) -> Optional[str]: + """Explain why no offers are available by refetching with unavailable offers included. + + Return "No quota" or "No balance" only if all matching offers have that reason. + Return None otherwise or if the lookup fails. + + Adds one offer lookup only when no available offers were found. Backend loading and + quota/balance checks may repeat DB reads; provider cache misses may repeat + provider calls. + """ + # TODO: Return JobTerminationReason values and persist + # FAILED_TO_START_DUE_TO_NO_QUOTA/NO_BALANCE once all supported rolling-upgrade + # source versions recognize them. Until then, use messages so older replicas + # can read the shared DB. + # Refetching will still be needed: the initial lookup excludes unavailable offers. + try: + offers = await get_offers_by_requirements( + project=project, + profile=profile, + requirements=requirements, + exclude_not_available=False, + multinode=requirements.multinode or is_multinode_job(job), + master_job_provisioning_data=master_job_provisioning_data, + volumes=volumes, + privileged=job.job_spec.privileged, + instance_mounts=check_run_spec_requires_instance_mounts(run.run_spec), + placement_group=placement_group_model_to_placement_group_optional( + placement_group_model + ), + ) + return await run_async(get_offers_termination_reason_message, offers) + except Exception: + logger.warning("%s: failed to check unavailable offers", fmt(job_model), exc_info=True) + return None + + _MAX_OFFER_ERROR_LEN = 200 diff --git a/src/dstack/_internal/server/background/pipeline_tasks/runs/common.py b/src/dstack/_internal/server/background/pipeline_tasks/runs/common.py index 03c528c9c1..ff30950808 100644 --- a/src/dstack/_internal/server/background/pipeline_tasks/runs/common.py +++ b/src/dstack/_internal/server/background/pipeline_tasks/runs/common.py @@ -127,7 +127,7 @@ async def delete_superseded_no_capacity_job_submissions( new_job_models: list[JobModel], ) -> None: """ - Delete previous job submissions that failed to start due to no capacity without + Delete previous job submissions that failed to start due to capacity, quota, or balance without ever provisioning. Such submissions are created each time a run is retrying on no capacity and carry no information beyond the events. The direct predecessor of each new submission is kept so that the run's @@ -143,8 +143,13 @@ async def delete_superseded_no_capacity_job_submissions( JobModel.job_num == job_model.job_num, JobModel.submission_num < job_model.submission_num - 1, JobModel.status.in_(JobStatus.finished_statuses()), - JobModel.termination_reason - == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + JobModel.termination_reason.in_( + [ + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_QUOTA, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_BALANCE, + ] + ), JobModel.job_provisioning_data.is_(None), ) ) diff --git a/src/dstack/_internal/server/services/jobs/__init__.py b/src/dstack/_internal/server/services/jobs/__init__.py index 0326be9aca..1ebbeecf9b 100644 --- a/src/dstack/_internal/server/services/jobs/__init__.py +++ b/src/dstack/_internal/server/services/jobs/__init__.py @@ -9,7 +9,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import load_only -from dstack._internal.core.consts import DSTACK_RUNNER_HTTP_PORT +from dstack._internal.core.consts import ( + DSTACK_RUNNER_HTTP_PORT, + NO_BALANCE_MESSAGE, + NO_QUOTA_MESSAGE, +) from dstack._internal.core.errors import ( ResourceNotExistsError, ServerClientError, @@ -625,7 +629,15 @@ def _get_job_status_message(job_model: JobModel) -> str: and "No matching fleet found" in job_model.termination_reason_message ): return "no fleets" + if job_model.termination_reason_message in (NO_QUOTA_MESSAGE, NO_BALANCE_MESSAGE): + return job_model.termination_reason_message.lower() return "no capacity" + elif job_model.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_QUOTA: + return "no quota" + elif ( + job_model.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_BALANCE + ): + return "no balance" elif job_model.termination_reason == JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY: return "interrupted" else: diff --git a/src/dstack/_internal/server/services/offers.py b/src/dstack/_internal/server/services/offers.py index eb826e03ee..1e1342ff6f 100644 --- a/src/dstack/_internal/server/services/offers.py +++ b/src/dstack/_internal/server/services/offers.py @@ -13,8 +13,10 @@ BACKENDS_WITH_PRIVILEGED_SUPPORT, BACKENDS_WITH_RESERVATION_SUPPORT, ) +from dstack._internal.core.consts import NO_BALANCE_MESSAGE, NO_QUOTA_MESSAGE from dstack._internal.core.models.backends.base import BackendType from dstack._internal.core.models.instances import ( + InstanceAvailability, InstanceOfferWithAvailability, InstanceType, Resources, @@ -129,6 +131,17 @@ async def get_offers_by_requirements( return await run_async(_sort_offers_by_availability, offers) +def get_offers_termination_reason_message( + offers: Iterable[Tuple[Backend, InstanceOfferWithAvailability]], +) -> Optional[str]: + availabilities = {offer.availability for _, offer in offers} + if availabilities == {InstanceAvailability.NO_QUOTA}: + return NO_QUOTA_MESSAGE + if availabilities == {InstanceAvailability.NO_BALANCE}: + return NO_BALANCE_MESSAGE + return None + + def _sort_offers_by_availability( offers: Iterable[Tuple[Backend, InstanceOfferWithAvailability]], ) -> List[Tuple[Backend, InstanceOfferWithAvailability]]: diff --git a/src/dstack/_internal/server/services/runs/__init__.py b/src/dstack/_internal/server/services/runs/__init__.py index b6ff199641..336d11fb38 100644 --- a/src/dstack/_internal/server/services/runs/__init__.py +++ b/src/dstack/_internal/server/services/runs/__init__.py @@ -1132,7 +1132,11 @@ def _get_run_status_message(run_model: RunModel, job_models: List[JobModel]) -> last_job_termination_reason = _get_last_job_termination_reason(job_models) if ( last_job_termination_reason - == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY + in ( + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_QUOTA, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_BALANCE, + ) and RetryEvent.NO_CAPACITY in retry_on_events ): # TODO: Show `retrying` for other retry events diff --git a/src/tests/_internal/cli/utils/test_fleet.py b/src/tests/_internal/cli/utils/test_fleet.py index f8a6254758..06e534c4f1 100644 --- a/src/tests/_internal/cli/utils/test_fleet.py +++ b/src/tests/_internal/cli/utils/test_fleet.py @@ -3,6 +3,7 @@ from typing import List, Optional from uuid import uuid4 +import pytest from rich.table import Table from rich.text import Text @@ -24,6 +25,7 @@ Gpu, Instance, InstanceStatus, + InstanceTerminationReason, InstanceType, Resources, SSHKey, @@ -214,6 +216,22 @@ def create_ssh_fleet( class TestGetFleetsTable: + @pytest.mark.parametrize( + ("message", "expected"), + [("No quota", "no quota"), ("No balance", "no balance"), ("Other error", "no_offers")], + ) + def test_blocked_offers_with_legacy_reason(self, message, expected): + instance = create_test_instance(status=InstanceStatus.TERMINATED) + instance.termination_reason = InstanceTerminationReason.NO_OFFERS.value + instance.termination_reason_message = message + fleet = create_backend_fleet(instances=[instance]) + + cells = get_table_cells( + get_fleets_table([fleet], current_project="test-project", verbose=True) + ) + + assert cells[1]["ERROR"] == expected + def test_backend_fleet_without_verbose(self): instance = create_test_instance( instance_num=0, diff --git a/src/tests/_internal/cli/utils/test_run.py b/src/tests/_internal/cli/utils/test_run.py index dffde505e8..2f1044bab7 100644 --- a/src/tests/_internal/cli/utils/test_run.py +++ b/src/tests/_internal/cli/utils/test_run.py @@ -258,6 +258,22 @@ async def test_simple_run(self, session: AsyncSession): "no capacity", "gold1", ), + ( + JobStatus.FAILED, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_QUOTA, + None, + "All matching offers are blocked by quota", + "no quota", + "indian_red1", + ), + ( + JobStatus.FAILED, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_BALANCE, + None, + "All matching offers are blocked by insufficient balance", + "no balance", + "indian_red1", + ), ( JobStatus.FAILED, JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, diff --git a/src/tests/_internal/core/models/test_runs.py b/src/tests/_internal/core/models/test_runs.py index e911a07125..170f450d3b 100644 --- a/src/tests/_internal/core/models/test_runs.py +++ b/src/tests/_internal/core/models/test_runs.py @@ -83,6 +83,8 @@ def test_get_error_returns_expected_messages(): # already handled and shown in status_message no_error_reasons = [ JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_QUOTA, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_BALANCE, JobTerminationReason.INTERRUPTED_BY_NO_CAPACITY, JobTerminationReason.WAITING_RUNNER_LIMIT_EXCEEDED, JobTerminationReason.TERMINATED_BY_USER, diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_fleets.py b/src/tests/_internal/server/background/pipeline_tasks/test_fleets.py index 2268ad9400..47e2c92b76 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_fleets.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_fleets.py @@ -693,8 +693,20 @@ async def test_prefers_surviving_instance_over_new_replacement_for_master_electi for instance in non_deleted_instances ) + @pytest.mark.parametrize( + "reason", + [ + InstanceTerminationReason.NO_OFFERS, + InstanceTerminationReason.NO_QUOTA, + InstanceTerminationReason.NO_BALANCE, + ], + ) async def test_min_zero_failed_master_terminates_unprovisioned_siblings( - self, test_db, session: AsyncSession, worker: FleetWorker + self, + test_db, + session: AsyncSession, + worker: FleetWorker, + reason: InstanceTerminationReason, ): project = await create_project(session) fleet = await create_fleet( @@ -716,7 +728,7 @@ async def test_min_zero_failed_master_terminates_unprovisioned_siblings( offer=None, instance_num=0, ) - failed_master.termination_reason = InstanceTerminationReason.NO_OFFERS + failed_master.termination_reason = reason sibling1 = await create_instance( session=session, project=project, diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_cloud_provisioning.py b/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_cloud_provisioning.py index 45f29c8976..b8c8213c16 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_cloud_provisioning.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_instances/test_cloud_provisioning.py @@ -130,6 +130,7 @@ async def test_creates_instance( assert instance.status == InstanceStatus.PROVISIONING assert instance.total_blocks == expected_blocks assert instance.busy_blocks == 0 + backend_mock.compute.return_value.get_offers.assert_called_once() @pytest.mark.parametrize("err", [RuntimeError("Unexpected"), ProvisioningError("Expected")]) async def test_tries_second_offer_if_first_fails( @@ -197,6 +198,8 @@ async def test_fails_if_all_offers_fail( await session.refresh(instance) assert instance.status == InstanceStatus.TERMINATED assert instance.termination_reason == InstanceTerminationReason.NO_OFFERS + assert instance.termination_reason_message == "All offers failed" + aws_mock.compute.return_value.get_offers.assert_called_once() async def test_fails_if_no_offers( self, @@ -217,6 +220,50 @@ async def test_fails_if_no_offers( await session.refresh(instance) assert instance.status == InstanceStatus.TERMINATED assert instance.termination_reason == InstanceTerminationReason.NO_OFFERS + assert instance.termination_reason_message == "No offers found" + + @pytest.mark.parametrize( + ("availabilities", "message"), + [ + ([InstanceAvailability.NO_QUOTA] * 2, "No quota"), + ( + [InstanceAvailability.NO_QUOTA, InstanceAvailability.NOT_AVAILABLE], + "No offers found", + ), + ([InstanceAvailability.NO_BALANCE] * 2, "No balance"), + ], + ) + async def test_fails_without_attempting_unavailable_offers( + self, + test_db, + session: AsyncSession, + worker: InstanceWorker, + availabilities: list[InstanceAvailability], + message: str, + ): + project = await create_project(session=session) + instance = await create_instance( + session=session, project=project, status=InstanceStatus.PENDING + ) + backend = Mock(TYPE=BackendType.DSTACK) + compute = Mock(spec=ComputeMockSpec) + backend.compute.return_value = compute + compute.get_offers.return_value = [ + get_instance_offer_with_availability(availability=availability) + for availability in availabilities + ] + with patch( + "dstack._internal.server.services.backends.get_project_backends", + return_value=[backend], + ): + await process_instance(session, worker, instance) + + await session.refresh(instance) + assert instance.status == InstanceStatus.TERMINATED + assert instance.termination_reason == InstanceTerminationReason.NO_OFFERS + assert instance.termination_reason_message == message + assert compute.get_offers.call_count == 2 + compute.create_instance.assert_not_called() async def test_waits_when_fleet_has_no_current_master( self, diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_runs/test_pending.py b/src/tests/_internal/server/background/pipeline_tasks/test_runs/test_pending.py index 6cb006821d..163f62ee2b 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_runs/test_pending.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_runs/test_pending.py @@ -154,8 +154,16 @@ async def test_resubmits_retrying_run_after_delay( assert new_job.replica_num == old_job.replica_num assert new_job.submission_num == old_job.submission_num + 1 + @pytest.mark.parametrize( + "reason", + [ + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_QUOTA, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_BALANCE, + ], + ) async def test_resubmission_deletes_superseded_no_capacity_submissions( - self, test_db, session: AsyncSession, worker: RunWorker + self, test_db, session: AsyncSession, worker: RunWorker, reason: JobTerminationReason ) -> None: """ On resubmission, older submissions that failed to start due to no capacity @@ -180,7 +188,7 @@ async def test_resubmission_deletes_superseded_no_capacity_submissions( status=JobStatus.FAILED, last_processed_at=old_time, submission_num=0, - termination_reason=JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + termination_reason=reason, ) provisioned_job = await create_job( session=session, @@ -188,7 +196,7 @@ async def test_resubmission_deletes_superseded_no_capacity_submissions( status=JobStatus.FAILED, last_processed_at=old_time, submission_num=1, - termination_reason=JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + termination_reason=reason, job_provisioning_data=get_job_provisioning_data(), ) other_reason_job = await create_job( @@ -205,7 +213,7 @@ async def test_resubmission_deletes_superseded_no_capacity_submissions( status=JobStatus.FAILED, last_processed_at=old_time, submission_num=3, - termination_reason=JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + termination_reason=reason, ) lock_run(run) await session.commit() diff --git a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py index 536df52cc1..ff83046e09 100644 --- a/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py +++ b/src/tests/_internal/server/background/pipeline_tasks/test_submitted_jobs.py @@ -25,7 +25,7 @@ ) from dstack._internal.core.models.envs import Env from dstack._internal.core.models.fleets import FleetNodesSpec, InstanceGroupPlacement -from dstack._internal.core.models.instances import InstanceStatus +from dstack._internal.core.models.instances import InstanceAvailability, InstanceStatus from dstack._internal.core.models.placement import PlacementGroup from dstack._internal.core.models.profiles import ( CreationPolicy, @@ -2061,6 +2061,7 @@ async def test_reports_tried_offers_when_new_capacity_provisioning_fails( # The first pass assigns the job to the fleet, the second one provisions. await _process_job(session=session, worker=worker, job_model=job) + compute_mock.get_offers.reset_mock() await _process_job(session=session, worker=worker, job_model=job) job = await _get_job(session, job.id) @@ -2069,6 +2070,99 @@ async def test_reports_tried_offers_when_new_capacity_provisioning_fails( assert f"Failed to provision in fleet '{fleet.name}'" in job.termination_reason_message assert "tried 1 of 1 offers, all failed" in job.termination_reason_message assert "us-east-1: InsufficientInstanceCapacity" in job.termination_reason_message + compute_mock.get_offers.assert_called_once() + + @pytest.mark.parametrize( + ("availabilities", "message"), + [ + ([], None), + ( + [InstanceAvailability.NO_QUOTA] * 2, + "No quota", + ), + ( + [InstanceAvailability.NO_QUOTA, InstanceAvailability.NOT_AVAILABLE], + None, + ), + ( + [InstanceAvailability.NO_BALANCE], + "No balance", + ), + ], + ) + async def test_fails_without_attempting_unavailable_offers( + self, + test_db, + session: AsyncSession, + worker: JobSubmittedWorker, + availabilities: list[InstanceAvailability], + message: str | None, + ): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + fleet = await create_fleet(session=session, project=project) + run = await create_run(session=session, project=project, repo=repo, user=user, fleet=fleet) + job = await create_job(session=session, run=run) + backend = Mock(TYPE=BackendType.DSTACK) + compute = Mock(spec=ComputeMockSpec) + backend.compute.return_value = compute + compute.get_offers.return_value = [ + get_instance_offer_with_availability(availability=availability) + for availability in availabilities + ] + with patch( + "dstack._internal.server.services.backends.get_project_backends", + return_value=[backend], + ): + await _process_job(session=session, worker=worker, job_model=job) + await _process_job(session=session, worker=worker, job_model=job) + + job = await _get_job(session, job.id) + assert job.status == JobStatus.TERMINATING + assert job.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY + if message is not None: + assert job.termination_reason_message == message + else: + assert job.termination_reason_message is not None + assert "No offers matching the run requirements" in job.termination_reason_message + assert compute.get_offers.call_count == 2 + compute.run_job.assert_not_called() + compute.run_jobs.assert_not_called() + compute.create_instance.assert_not_called() + + async def test_keeps_generic_failure_if_refetch_fails( + self, test_db, session: AsyncSession, worker: JobSubmittedWorker + ): + project = await create_project(session=session) + user = await create_user(session=session) + repo = await create_repo(session=session, project_id=project.id) + fleet = await create_fleet(session=session, project=project) + run = await create_run(session=session, project=project, repo=repo, user=user, fleet=fleet) + job = await create_job(session=session, run=run) + backend = Mock(TYPE=BackendType.DSTACK) + compute = Mock(spec=ComputeMockSpec) + backend.compute.return_value = compute + + compute.get_offers.return_value = [ + get_instance_offer_with_availability(availability=InstanceAvailability.NO_QUOTA) + ] + with patch( + "dstack._internal.server.services.backends.get_project_backends", + side_effect=[[backend], RuntimeError("Diagnostic lookup failed")], + ): + await _process_job(session=session, worker=worker, job_model=job) + await _process_job(session=session, worker=worker, job_model=job) + + job = await _get_job(session, job.id) + assert job.status == JobStatus.TERMINATING + assert job.termination_reason == JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY + assert job.termination_reason_message is not None + assert "No offers matching the run requirements" in job.termination_reason_message + compute.get_offers.assert_called_once() + compute.run_job.assert_not_called() + compute.run_jobs.assert_not_called() + compute.create_instance.assert_not_called() async def test_provisions_compute_group( self, test_db, session: AsyncSession, worker: JobSubmittedWorker diff --git a/src/tests/_internal/server/services/jobs/test_jobs.py b/src/tests/_internal/server/services/jobs/test_jobs.py index 96b127e823..93a083f1a0 100644 --- a/src/tests/_internal/server/services/jobs/test_jobs.py +++ b/src/tests/_internal/server/services/jobs/test_jobs.py @@ -10,11 +10,13 @@ from dstack._internal.core.models.profiles import Profile from dstack._internal.core.models.repos.local import LocalRunRepoData from dstack._internal.core.models.resources import ResourcesSpec -from dstack._internal.core.models.runs import JobSpec, RunSpec +from dstack._internal.core.models.runs import JobSpec, JobStatus, JobTerminationReason, RunSpec from dstack._internal.core.models.volumes import DaytonaVolumeConfiguration +from dstack._internal.server.models import JobModel from dstack._internal.server.services.docker import ImageConfig from dstack._internal.server.services.jobs import ( _get_job_mount_point_attached_volume, + _get_job_status_message, get_job_specs_from_run_spec, job_spec_updatable_in_place, ) @@ -26,6 +28,20 @@ ) +class TestGetJobStatusMessage: + @pytest.mark.parametrize( + ("message", "expected"), + [("No quota", "no quota"), ("No balance", "no balance"), ("Other error", "no capacity")], + ) + def test_reports_blocked_offers_with_legacy_reason(self, message, expected): + job = JobModel( + status=JobStatus.FAILED, + termination_reason=JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + termination_reason_message=message, + ) + assert _get_job_status_message(job) == expected + + @pytest.mark.parametrize( "configuration, expected_calls", [ diff --git a/src/tests/_internal/server/services/runs/test_runs.py b/src/tests/_internal/server/services/runs/test_runs.py index cc283a6812..6c624f1353 100644 --- a/src/tests/_internal/server/services/runs/test_runs.py +++ b/src/tests/_internal/server/services/runs/test_runs.py @@ -27,11 +27,20 @@ class TestListUserRuns: @pytest.mark.asyncio @pytest.mark.parametrize("test_db", ["sqlite", "postgres"], indirect=True) @pytest.mark.parametrize("include_jobs", [True, False]) + @pytest.mark.parametrize( + "termination_reason", + [ + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_QUOTA, + JobTerminationReason.FAILED_TO_START_DUE_TO_NO_BALANCE, + ], + ) async def test_limited_list_materializes_only_latest_and_status_jobs( self, test_db, session: AsyncSession, include_jobs: bool, + termination_reason: JobTerminationReason, monkeypatch: pytest.MonkeyPatch, ) -> None: user = await create_user(session=session, global_role=GlobalRole.USER) @@ -60,7 +69,7 @@ async def test_limited_list_materializes_only_latest_and_status_jobs( run=run, submission_num=0, status=JobStatus.TERMINATED, - termination_reason=JobTerminationReason.FAILED_TO_START_DUE_TO_NO_CAPACITY, + termination_reason=termination_reason, ) for submission_num in range(1, 12): await create_job(