Skip to content

First version of the meister profiler #561

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
juandiegopalomino wants to merge 1 commit into
base: main
Choose a base branch
from
Open

First version of the meister profiler #561

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions opta/cli.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from colored import attr, fg

import opta.sentry # noqa: F401 This leads to initialization of sentry sdk
from opta import meister
from opta.cleanup_files import cleanup_files
from opta.commands.apply import apply
from opta.commands.deploy import deploy
Expand All @@ -24,6 +25,7 @@
from opta.commands.shell import shell
from opta.commands.validate import validate
from opta.commands.version import version
from opta.constants import DEV_VERSION, VERSION
from opta.crash_reporter import CURRENT_CRASH_REPORTER
from opta.exceptions import UserErrors
from opta.one_time import one_time
Expand Down Expand Up @@ -61,6 +63,7 @@ def cli() -> None:
# after the command.
# However, we should still clean them up before the next command, or
# else it may interfere with it.
meister.REPORT_ENABLED = VERSION in [DEV_VERSION, ""]
Copy link
Collaborator

@NitinAgg NitinAgg Jan 14, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's make this based on an env flag instead. I don't want outside contributors to run into this.

one_time()
cleanup_files()
cli()
Expand Down
4 changes: 4 additions & 0 deletions opta/commands/apply.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
from opta.error_constants import USER_ERROR_TF_LOCK
from opta.exceptions import MissingState, UserErrors
from opta.layer import Layer, StructuredConfig
from opta.meister import time as meister_time
from opta.pre_check import pre_check
from opta.utils import check_opta_file_exists, fmt_msg, logger

Expand Down Expand Up @@ -329,6 +330,7 @@ def _apply(
)


@meister_time
def _verify_semver(
old_semver_string: str,
current_semver_string: str,
Expand Down Expand Up @@ -374,6 +376,7 @@ def _verify_semver(


# Fetch the AZs of a region with boto3
@meister_time
def _fetch_availability_zones(aws_region: str) -> List[str]:
client = boto3.client("ec2", config=Config(region_name=aws_region))
azs: List[str] = []
Expand All @@ -385,6 +388,7 @@ def _fetch_availability_zones(aws_region: str) -> List[str]:


# Verify whether the parent layer exists or not
@meister_time
def _verify_parent_layer(layer: Layer, auto_approve: bool = False) -> None:
if layer.parent is None:
return
Expand Down
6 changes: 6 additions & 0 deletions opta/commands/destroy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from opta.error_constants import USER_ERROR_TF_LOCK
from opta.exceptions import UserErrors
from opta.layer import Layer
from opta.meister import time as meister_time
from opta.pre_check import pre_check
from opta.utils import check_opta_file_exists, logger

Expand Down Expand Up @@ -108,6 +109,7 @@ def destroy(


# Fetch all the children layers of the current layer.
@meister_time
def _fetch_children_layers(layer: "Layer") -> List[str]:
# Only environment layers have children (service) layers.
# If the current layer has a parent, it is *not* an environment layer.
Expand All @@ -130,6 +132,7 @@ def _fetch_children_layers(layer: "Layer") -> List[str]:


# Get the names for all services for this environment based on the bucket file paths
@meister_time
def _azure_get_configs(layer: "Layer") -> List[str]:
providers = layer.gen_providers(0)

Expand All @@ -151,6 +154,7 @@ def _azure_get_configs(layer: "Layer") -> List[str]:
return configs


@meister_time
def _aws_get_configs(layer: "Layer") -> List[str]:
# Opta configs for every layer are saved in the opta_config/ directory
# in the state bucket.
Expand All @@ -168,6 +172,7 @@ def _aws_get_configs(layer: "Layer") -> List[str]:
return configs


@meister_time
def _gcp_get_configs(layer: "Layer") -> List[str]:
bucket_name = layer.state_storage()
gcs_config_dir = "opta_config/"
Expand All @@ -189,6 +194,7 @@ def _gcp_get_configs(layer: "Layer") -> List[str]:
return configs


@meister_time
def _local_get_configs(layer: "Layer") -> List[str]:
local_config_dir = os.path.join(
os.path.join(str(Path.home()), ".opta", "local", "opta_config")
Expand Down
3 changes: 3 additions & 0 deletions opta/commands/local_flag.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@

from ruamel import yaml

from opta.meister import time as meister_time
from opta.utils import logger


@meister_time
def _handle_local_flag(config: str, test: bool = False) -> str:
if test:
return config
Expand Down Expand Up @@ -41,6 +43,7 @@ def _handle_local_flag(config: str, test: bool = False) -> str:
return config


@meister_time
def _clean_tf_folder() -> None:
if os.path.isdir(os.getcwd() + "/.terraform"):
rmtree(os.getcwd() + "/.terraform")
4 changes: 3 additions & 1 deletion opta/commands/output.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from opta.core.generator import gen_all
from opta.core.terraform import get_terraform_outputs
from opta.layer import Layer
from opta.meister import time as meister_time
from opta.utils import check_opta_file_exists, json


Expand All @@ -16,7 +17,6 @@
)
def output(config: str, env: Optional[str],) -> None:
"""Print TF outputs"""

config = check_opta_file_exists(config)
layer = Layer.load_from_yaml(config, env)
amplitude_client.send_event(
Expand All @@ -35,6 +35,7 @@ def output(config: str, env: Optional[str],) -> None:
print(outputs_formatted)


@meister_time
def _load_extra_aws_outputs(current_outputs: dict) -> dict:
if "parent.load_balancer_raw_dns" in current_outputs:
current_outputs["load_balancer_raw_dns"] = current_outputs[
Expand All @@ -44,6 +45,7 @@ def _load_extra_aws_outputs(current_outputs: dict) -> dict:
return current_outputs


@meister_time
def _load_extra_gcp_outputs(current_outputs: dict) -> dict:
if "parent.load_balancer_raw_ip" in current_outputs:
current_outputs["load_balancer_raw_ip"] = current_outputs[
Expand Down
10 changes: 10 additions & 0 deletions opta/commands/push.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,13 @@
from opta.core.terraform import get_terraform_outputs
from opta.exceptions import UserErrors
from opta.layer import Layer
from opta.meister import time as meister_time
from opta.nice_subprocess import nice_run
from opta.utils import check_opta_file_exists, fmt_msg, yaml
from opta.utils.dependencies import ensure_installed


@meister_time
def get_push_tag(local_image: str, tag_override: Optional[str]) -> str:
if ":" not in local_image:
raise Exception(
Expand All @@ -26,6 +28,7 @@ def get_push_tag(local_image: str, tag_override: Optional[str]) -> str:
return tag_override or local_image_tag


@meister_time
def get_image_digest(registry_url: str, image_tag: str) -> str:
docker_client = from_env()
current_image = docker_client.images.get(f"{registry_url}:{image_tag}")
Expand All @@ -42,6 +45,7 @@ def get_image_digest(registry_url: str, image_tag: str) -> str:
)


@meister_time
def get_registry_url(layer: Layer) -> str:
outputs = get_terraform_outputs(layer)
if "docker_repo_url" not in outputs:
Expand All @@ -52,6 +56,7 @@ def get_registry_url(layer: Layer) -> str:
return outputs["docker_repo_url"]


@meister_time
def get_ecr_auth_info(layer: Layer) -> Tuple[str, str]:
providers = layer.gen_providers(0)
account_id = providers["provider"]["aws"]["allowed_account_ids"][0]
Expand All @@ -73,6 +78,7 @@ def get_ecr_auth_info(layer: Layer) -> Tuple[str, str]:
return username, password


@meister_time
def get_gcr_auth_info(layer: Layer) -> Tuple[str, str]:
if GCP.using_service_account():
service_account_key = GCP.get_service_account_raw_credentials()
Expand All @@ -82,6 +88,7 @@ def get_gcr_auth_info(layer: Layer) -> Tuple[str, str]:
return "oauth2accesstoken", credentials.token


@meister_time
def get_acr_auth_info(layer: Layer) -> Tuple[str, str]:
acr_name = get_terraform_outputs(layer.root()).get("acr_name")
if acr_name is None:
Expand All @@ -105,6 +112,7 @@ def get_acr_auth_info(layer: Layer) -> Tuple[str, str]:
return "00000000-0000-0000-0000-000000000000", token


@meister_time
def push_to_docker(
username: str,
password: str,
Expand All @@ -124,6 +132,7 @@ def push_to_docker(
return get_image_digest(registry_url, image_tag), image_tag


@meister_time
def push_to_docker_local(
local_image: str, registry_url: str, image_tag_override: Optional[str],
) -> Tuple[str, str]:
Expand Down Expand Up @@ -173,6 +182,7 @@ def push(image: str, config: str, env: Optional[str], tag: Optional[str]) -> Non
_push(image, config, env, tag)


@meister_time
def _push(
image: str, config: str, env: Optional[str], tag: Optional[str]
) -> Tuple[str, str]:
Expand Down
Loading