diff --git a/Dockerfile b/Dockerfile index 6b545ef72767..9e3daf6a02a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,10 @@ FROM python:3.11-slim +# Install cURL +RUN apt-get update && \ + apt-get install -y curl && \ + rm -rf /var/lib/apt/lists/* + ADD . /moto/ ENV PYTHONUNBUFFERED 1 @@ -7,10 +12,6 @@ WORKDIR /moto/ RUN pip3 --no-cache-dir install --upgrade pip setuptools && \ pip3 --no-cache-dir install ".[server]" -# Install cURL -RUN apt-get update && \ - apt-get install -y curl && \ - rm -rf /var/lib/apt/lists/* ENTRYPOINT ["/usr/local/bin/moto_server", "-H", "0.0.0.0"] diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 803d55e673ce..63dd6484ae27 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -481,7 +481,8 @@ def __init__( ) self.suspended_processes: List[str] = [] - self.instance_states: List[InstanceState] = [] + # Performance patch: replace list with dict instance id -> instance state + self.instance_states: Dict[str, InstanceState] = {} self.tags: List[Dict[str, str]] = tags or [] self.set_desired_capacity(desired_capacity) @@ -507,7 +508,9 @@ def arn(self) -> str: return f"arn:{get_partition(self.region)}:autoscaling:{self.region}:{self.account_id}:autoScalingGroup:{self._id}:autoScalingGroupName/{self.name}" def active_instances(self) -> List[InstanceState]: - return [x for x in self.instance_states if x.lifecycle_state == "InService"] + return [ + x for x in self.instance_states.values() if x.lifecycle_state == "InService" + ] def _set_azs_and_vpcs( self, @@ -790,7 +793,7 @@ def set_desired_capacity(self, new_capacity: Optional[int]) -> None: count_to_remove = curr_instance_count - self.desired_capacity # type: ignore[operator] instances_to_remove = [ # only remove unprotected state - for state in self.instance_states + for state in self.instance_states.values() if not state.protected_from_scale_in ][:count_to_remove] if instances_to_remove: # just in case not instances to remove @@ -800,9 +803,11 @@ def set_desired_capacity(self, new_capacity: Optional[int]) -> None: self.autoscaling_backend.ec2_backend.terminate_instances( instance_ids_to_remove ) - self.instance_states = list( - set(self.instance_states) - set(instances_to_remove) - ) + self.instance_states = { + state.instance.id: state + for state in set(self.instance_states.values()) + - set(instances_to_remove) + } if self.name in self.autoscaling_backend.autoscaling_groups: self.autoscaling_backend.update_attached_elbs(self.name) self.autoscaling_backend.update_attached_target_groups(self.name) @@ -869,11 +874,9 @@ def replace_autoscaling_group_instances( ) for instance in reservation.instances: instance.autoscaling_group = self - self.instance_states.append( - InstanceState( - instance, - protected_from_scale_in=self.new_instances_protected_from_scale_in, - ) + self.instance_states[instance.id] = InstanceState( + instance, + protected_from_scale_in=self.new_instances_protected_from_scale_in, ) def append_target_groups(self, target_group_arns: List[str]) -> None: @@ -1219,7 +1222,14 @@ def update_auto_scaling_group( def describe_auto_scaling_groups( self, names: List[str], filters: Optional[List[Dict[str, str]]] = None ) -> List[FakeAutoScalingGroup]: - groups = list(self.autoscaling_groups.values()) + if names: + groups = [ + self.autoscaling_groups[name] + for name in names + if name in self.autoscaling_groups + ] + else: + groups = list(self.autoscaling_groups.values()) if filters: for f in filters: @@ -1246,9 +1256,6 @@ def describe_auto_scaling_groups( ) ] - if names: - groups = [group for group in groups if group.name in names] - return groups def delete_auto_scaling_group(self, group_name: str) -> None: @@ -1260,13 +1267,16 @@ def describe_auto_scaling_instances( ) -> List[InstanceState]: instance_states = [] for group in self.autoscaling_groups.values(): - instance_states.extend( - [ - x - for x in group.instance_states - if not instance_ids or x.instance.id in instance_ids - ] - ) + if not instance_ids: + instance_states.extend(group.instance_states.values()) + continue + for instance_id in instance_ids: + try: + instance_states.append(group.instance_states[instance_id]) + except KeyError: + pass + if len(instance_states) == len(instance_ids): + break return instance_states def attach_instances(self, group_name: str, instance_ids: List[str]) -> None: @@ -1289,7 +1299,9 @@ def attach_instances(self, group_name: str, instance_ids: List[str]) -> None: self.ec2_backend.create_tags( [instance.instance.id], {ASG_NAME_TAG: group.name} ) - group.instance_states.extend(new_instances) + group.instance_states.update( + (instance.instance.id, instance) for instance in new_instances + ) self.update_attached_elbs(group.name) self.update_attached_target_groups(group.name) @@ -1297,12 +1309,10 @@ def set_instance_health(self, instance_id: str, health_status: str) -> None: """ The ShouldRespectGracePeriod-parameter is not yet implemented """ - instance = self.ec2_backend.get_instance(instance_id) instance_state = next( - instance_state + group.instance_states[instance_id] for group in self.autoscaling_groups.values() - for instance_state in group.instance_states - if instance_state.instance.id == instance.id + if instance_id in group.instance_states ) instance_state.health_status = health_status @@ -1313,14 +1323,11 @@ def detach_instances( original_size = group.desired_capacity detached_instances = [ - x for x in group.instance_states if x.instance.id in instance_ids + group.instance_states.pop(x) + for x in instance_ids + if x in group.instance_states ] - new_instance_state = [ - x for x in group.instance_states if x.instance.id not in instance_ids - ] - group.instance_states = new_instance_state - if should_decrement: group.desired_capacity = original_size - len(instance_ids) # type: ignore[operator] @@ -1472,7 +1479,7 @@ def update_attached_elbs(self, group_name: str) -> None: def update_attached_target_groups(self, group_name: str) -> None: group = self.autoscaling_groups[group_name] - group_instance_ids = set(state.instance.id for state in group.instance_states) + group_instance_ids = set(group.instance_states) # no action necessary if target_group_arns is empty if not group.target_group_arns: @@ -1534,7 +1541,7 @@ def detach_load_balancers( self, group_name: str, load_balancer_names: List[str] ) -> None: group = self.autoscaling_groups[group_name] - group_instance_ids = set(state.instance.id for state in group.instance_states) + group_instance_ids = set(group.instance_states) elbs = self.elb_backend.describe_load_balancers(names=group.load_balancers) for elb in elbs: self.elb_backend.deregister_instances( @@ -1562,7 +1569,7 @@ def detach_load_balancer_target_groups( x for x in group.target_group_arns if x not in target_group_arns ] for target_group in target_group_arns: - asg_targets = [{"id": x.instance.id} for x in group.instance_states] + asg_targets = [{"id": x} for x in group.instance_states] self.elbv2_backend.deregister_targets(target_group, (asg_targets)) def suspend_processes(self, group_name: str, scaling_processes: List[str]) -> None: @@ -1600,7 +1607,7 @@ def set_instance_protection( ) -> None: group = self.autoscaling_groups[group_name] protected_instances = [ - x for x in group.instance_states if x.instance.id in instance_ids + i for x, i in group.instance_states.items() if x in instance_ids ] for instance in protected_instances: instance.protected_from_scale_in = protected_from_scale_in @@ -1611,12 +1618,8 @@ def notify_terminate_instances(self, instance_ids: List[str]) -> None: autoscaling_group, ) in self.autoscaling_groups.items(): original_active_instance_count = len(autoscaling_group.active_instances()) - autoscaling_group.instance_states = list( - filter( - lambda i_state: i_state.instance.id not in instance_ids, - autoscaling_group.instance_states, - ) - ) + for instance_id in instance_ids: + autoscaling_group.instance_states.pop(instance_id, None) difference = original_active_instance_count - len( autoscaling_group.active_instances() ) @@ -1632,7 +1635,7 @@ def enter_standby_instances( group = self.autoscaling_groups[group_name] original_size = group.desired_capacity standby_instances = [] - for instance_state in group.instance_states: + for instance_state in group.instance_states.values(): if instance_state.instance.id in instance_ids: instance_state.lifecycle_state = "Standby" standby_instances.append(instance_state) @@ -1647,7 +1650,7 @@ def exit_standby_instances( group = self.autoscaling_groups[group_name] original_size = group.desired_capacity standby_instances = [] - for instance_state in group.instance_states: + for instance_state in group.instance_states.values(): if instance_state.instance.id in instance_ids: instance_state.lifecycle_state = "InService" standby_instances.append(instance_state) @@ -1659,19 +1662,13 @@ def terminate_instance( self, instance_id: str, should_decrement: bool ) -> Tuple[InstanceState, Any, Any]: instance = self.ec2_backend.get_instance(instance_id) + group = instance.autoscaling_group # type: ignore[attr-defined] try: - instance_state = next( - instance_state - for group in self.autoscaling_groups.values() - for instance_state in group.instance_states - if instance_state.instance.id == instance.id - ) - except StopIteration: + instance_state = group.instance_states[instance_id] + except KeyError: # Maybe the VM has already been undeployed manually in EC2. # In such a case, AWS does not throw any error here. instance_state = InstanceState(instance, lifecycle_state="Terminated") - - group = instance.autoscaling_group # type: ignore[attr-defined] original_size = group.desired_capacity self.detach_instances(group.name, [instance.id], should_decrement) self.ec2_backend.terminate_instances([instance.id]) diff --git a/moto/autoscaling/responses.py b/moto/autoscaling/responses.py index a4d3ea360a66..e4e3d2ee77e4 100644 --- a/moto/autoscaling/responses.py +++ b/moto/autoscaling/responses.py @@ -892,7 +892,7 @@ def delete_warm_pool(self) -> str: {% endif %} - {% for instance_state in group.instance_states %} + {% for instance_state in group.instance_states.values() %} {{ instance_state.health_status }} {{ instance_state.instance.placement }} diff --git a/moto/core/utils.py b/moto/core/utils.py index 46baa38d84d8..1de0008526b9 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -3,12 +3,12 @@ import re import threading import time -from flask import current_app from gzip import compress, decompress from typing import Any, Callable, Dict, List, Optional, Tuple from urllib.parse import ParseResult, urlparse from botocore.exceptions import ClientError +from flask import current_app from ..settings import get_s3_custom_endpoints from .common_types import TYPE_RESPONSE @@ -93,7 +93,7 @@ def caller(reg: Any) -> str: class convert_to_flask_response(object): lock = threading.Lock() - + def __init__(self, callback: Callable[..., Any]): self.callback = callback @@ -128,12 +128,18 @@ def __call__(self, args: Any = None, **kwargs: Any) -> Any: response = Response(response=content, status=status, headers=headers) if request.method == "HEAD" and "content-length" in headers: response.headers["Content-Length"] = headers["content-length"] - + total_time = time.perf_counter() - start_time time_in_ms = int(total_time * 1000) - current_app.logger.info('%s ms %s %s %s', time_in_ms, request.method, request.path, dict(request.args)) - - return response + current_app.logger.info( + "%s ms %s %s %s", + time_in_ms, + request.method, + request.path, + dict(request.args), + ) + + return response class convert_flask_to_responses_response(object): diff --git a/moto/ec2/models/amis.py b/moto/ec2/models/amis.py index eef068224086..e8b9e76959b1 100644 --- a/moto/ec2/models/amis.py +++ b/moto/ec2/models/amis.py @@ -314,14 +314,14 @@ def _original_describe_images( # in our usecases, only two AMIs are used _cache_images = {} - def describe_images( - self, ami_ids=(), filters=None, exec_users=None, owners=None - ): + def describe_images(self, ami_ids=(), filters=None, exec_users=None, owners=None): cache_key = (tuple(ami_ids), tuple(filters.items())) if cache_key in self._cache_images: return self._cache_images[cache_key] else: - images = self._original_describe_images(ami_ids, filters, exec_users, owners) + images = self._original_describe_images( + ami_ids, filters, exec_users, owners + ) self._cache_images[cache_key] = images return images diff --git a/moto/ec2/models/elastic_block_store.py b/moto/ec2/models/elastic_block_store.py index e0c35e18e2d8..0804399bf2b5 100644 --- a/moto/ec2/models/elastic_block_store.py +++ b/moto/ec2/models/elastic_block_store.py @@ -1,3 +1,4 @@ +from collections import defaultdict from typing import Any, Dict, Iterable, List, Optional, Set from moto.core.common_models import CloudFormationModel @@ -267,7 +268,8 @@ def get_filter_value( class EBSBackend: def __init__(self) -> None: self.volumes: Dict[str, Volume] = {} - self.attachments: Dict[str, VolumeAttachment] = {} + # Performance patch: fill this dict with instance id -> volume attachments + self.attachments: Dict[str, List[VolumeAttachment]] = defaultdict(list) self.snapshots: Dict[str, Snapshot] = {} self.default_kms_key_id: str = "" @@ -323,12 +325,27 @@ def create_volume( def describe_volumes( self, volume_ids: Optional[List[str]] = None, filters: Any = None ) -> List[Volume]: - matches = list(self.volumes.values()) + filters = filters or {} if volume_ids: - matches = [vol for vol in matches if vol.id in volume_ids] + matches = [ + vol + for vol_id in volume_ids + if (vol := self.volumes.get(vol_id)) is not None + ] if len(volume_ids) > len(matches): unknown_ids = set(volume_ids) - set(matches) # type: ignore[arg-type] raise InvalidVolumeIdError(unknown_ids) + # performance patch: filter by attachment.instance-id using attachments dict + elif instance_ids := filters.pop("attachment.instance-id", []): + matches = [] + for instance_id in instance_ids: + if instance_id in self.attachments: + matches.extend( + attachment.volume + for attachment in self.attachments[instance_id] + ) + else: + matches = list(self.volumes.values()) if filters: matches = generic_filter(filters, matches) return matches @@ -391,6 +408,7 @@ def attach_volume( delete_on_termination=delete_on_termination, ) instance.block_device_mapping[device_path] = bdt + self.attachments[instance_id].append(volume.attachment) return volume.attachment def detach_volume( @@ -406,6 +424,9 @@ def detach_volume( try: del instance.block_device_mapping[device_path] + self.attachments[instance_id].remove(volume.attachment) + if not self.attachments[instance_id]: + del self.attachments[instance_id] except KeyError: raise InvalidVolumeDetachmentError(volume_id, instance_id, device_path) diff --git a/moto/ec2/models/instances.py b/moto/ec2/models/instances.py index 7b8500f6f33c..6b0efbeb0b83 100644 --- a/moto/ec2/models/instances.py +++ b/moto/ec2/models/instances.py @@ -614,12 +614,14 @@ def applies(self, filters: List[Dict[str, Any]]) -> bool: class InstanceBackend: def __init__(self) -> None: self.reservations: Dict[str, Reservation] = OrderedDict() + # Performance patch: reference instances by their id + self.id_to_instances: Dict[str, Instance] = {} def get_instance(self, instance_id: str) -> Instance: - for instance in self.all_instances(): - if instance.id == instance_id: - return instance - raise InvalidInstanceIdError(instance_id) + try: + return self.id_to_instances[instance_id] + except KeyError: + raise InvalidInstanceIdError(instance_id) def _original_run_instances( self, @@ -702,6 +704,7 @@ def _original_run_instances( self, image_id, user_data, security_groups, **kwargs ) new_reservation.instances.append(new_instance) + self.id_to_instances[new_instance.id] = new_instance new_instance.add_tags(instance_tags) block_device_mappings = None if "block_device_mappings" not in kwargs: @@ -756,8 +759,6 @@ def _original_run_instances( # instances are never deleted (they are kept in "terminated" state) # so there is no clean-up to do def run_instances(self, *args, **kwargs): - if not hasattr(self, 'id_to_instances'): - self._build_instance_dict() result = self._original_run_instances(*args, **kwargs) result.id_to_instances = {} for instance in result.instances: @@ -766,17 +767,9 @@ def run_instances(self, *args, **kwargs): instance.reservation_id = result.id # delete_on_termination is True in later version of moto # this fix a warning in bootstrapper when instance is undeployed - instance.block_device_mapping['/dev/sda1'].delete_on_termination = True + instance.block_device_mapping["/dev/sda1"].delete_on_termination = True return result - - def _build_instance_dict(self): - if not hasattr(self, 'id_to_instances'): - self.id_to_instances = {} - for reservation in self.reservations.values(): - for instance in reservation.instances: - self.id_to_instances[instance.id] = instance - instance.reservation_id = reservation.id def start_instances( self, instance_ids: List[str] @@ -893,11 +886,7 @@ def get_multi_instances_by_id(self, instance_ids, filters=None): # PATCHED - REPLACED FORMER VERSION def get_instance_by_id(self, instance_id: str): - try: - return self.id_to_instances.get(instance_id) - except AttributeError: # self.id_to_instances has not been defined yet - self._build_instance_dict() - return self.get_instance_by_id(instance_id) + return self.id_to_instances.get(instance_id) def get_reservations_by_instance_ids( self, instance_ids: List[str], filters: Any = None @@ -906,41 +895,30 @@ def get_reservations_by_instance_ids( associated with the given instance_ids. """ reservations = [] - for reservation in self.all_reservations(): - reservation_instance_ids = [ - instance.id for instance in reservation.instances - ] - matching_reservation = any( - instance_id in reservation_instance_ids for instance_id in instance_ids - ) - if matching_reservation: - reservation.instances = [ - instance - for instance in reservation.instances - if instance.id in instance_ids - ] - reservations.append(reservation) - found_instance_ids = [ - instance.id - for reservation in reservations - for instance in reservation.instances - ] - if len(found_instance_ids) != len(instance_ids): - invalid_id = list(set(instance_ids).difference(set(found_instance_ids)))[0] - raise InvalidInstanceIdError(invalid_id) + reservations = {} + for id_ in instance_ids: + instance = self.get_instance(id_) + try: + reservations[instance.reservation_id].instances.append(instance) + except KeyError: + reservation = copy.copy(self.reservations[instance.reservation_id]) + reservation.instances = [instance] + reservations[instance.reservation_id] = reservation + + reservations = list(reservations.values()) if filters is not None: reservations = filter_reservations(reservations, filters) return reservations # ORIGINAL FUNCTION - PATCHED BELOW # def describe_instances(self, filters: Any = None) -> List[Reservation]: - # return self.all_reservations(filters) + # return self.all_reservations(filters) # Perfomrance patch: describe_instances to avoid double iterations on reservations # This a mix of all_reservations() and filter_reservations() as used by describe_instances() - def describe_instances(self, filters=None): + def describe_instances(self, filters=None): filters = filters or {} - instance_ids = filters.get('instance-id', set()) + instance_ids = filters.pop("instance-id", set()) if instance_ids: reservations = {} for id_ in instance_ids: @@ -949,7 +927,9 @@ def describe_instances(self, filters=None): try: reservations[instance.reservation_id].instances.append(instance) except KeyError: - reservation = copy.copy(self.reservations[instance.reservation_id]) + reservation = copy.copy( + self.reservations[instance.reservation_id] + ) reservation.instances = [instance] reservations[instance.reservation_id] = reservation return list(reservations.values()) @@ -963,7 +943,7 @@ def describe_instances(self, filters=None): if new_instances: reservation = copy.copy(reservation) reservation.instances = new_instances - result.append(reservation) + result.append(reservation) return result def describe_instance_status( diff --git a/moto/ec2/models/tags.py b/moto/ec2/models/tags.py index 810df4bff1d8..f00dc6e0d3d6 100644 --- a/moto/ec2/models/tags.py +++ b/moto/ec2/models/tags.py @@ -58,7 +58,11 @@ def delete_tags(self, resource_ids: List[str], tags: Dict[str, str]) -> bool: # this method is called each time a resource is rendered in a response # That's the main optimisation! def describe_tags(self, filters): - if "resource-id" in filters and len(filters) == 1 and len(filters["resource-id"]) == 1: + if ( + "resource-id" in filters + and len(filters) == 1 + and len(filters["resource-id"]) == 1 + ): results = [] resource_id = filters["resource-id"][0] try: @@ -70,9 +74,7 @@ def describe_tags(self, filters): "resource_id": resource_id, "key": key, "value": value, - "resource_type": EC2_PREFIX_TO_RESOURCE[ - get_prefix(resource_id) - ], + "resource_type": EC2_PREFIX_TO_RESOURCE[get_prefix(resource_id)], } results.append(result) return results