MediaConvert (#7)

* wip

* rename EC2 IAM things

* wip

* sync media convert rule and inject into .env during build

* drop Elastic Transcoder logic

* wip it

* wip

* wip

* create SNS topic for mediaconvert

* skip mediaconvert steps if not configured

* detail some features and change transcoder in manifest

* SNS subscription for Mediaconvert

* create EventBridge rule and rule target

* fix exceptions

* ensure the default AWS profile is not used

* wip

* drop SNS and event bridge logic for mediaconvert

* drop more mediaconvert related logic

* drop WOULD_SKIP

* wip

Author: stevethomas

README.md

@@ -3,22 +3,71 @@
 > [!IMPORTANT]
 > This package is in active development - contributions are welcome!
 
-YOLO helps you deploy high-availability PHP applications on AWS.
+YOLO helps you deploy high-availability PHP applications to AWS.
 
-The CLI tool takes care of provisioning and configuring all required resources on AWS, coupled with build and deployment
+The CLI tool lives inside your Laravel app in `vendor/bin/yolo`, and takes care of provisioning and configuring all
+required resources on
+AWS, coupled with build and deployment
 commands to deploy applications to production from your local machine or CI pipeline.
 
+YOLO has been battle-tested on apps that serve 2 million requests per day.
+
+## Features
+
+### Autoscaling Worker Groups
+
+YOLO provisions an Application Load Balancer and autoscaling groups (web, queue, scheduler) for each environment.
+
+Each group is self-healing should an instance become unresponsive, and the web group automatically scales up to handle
+traffic bursts.
+
+In addition, worker groups can be combined (coming soon) to a single EC2 instance to consolidate small workloads.
+
+### Resource Sharing
+
+YOLO shares various resources between applications to reduce costs.
+
+### Zero-downtime Deployments
+
+YOLO leverages AWS CodeDeploy to perform zero-downtime deployments, which can be triggered from the CLI or via a CI
+pipeline.
+
+### Multi-tenancy
+
+Specify tenants in the manifest and YOLO will take care of provisioning resources for each tenant.
+
+### S3
+
+Leverage S3 for storing build artefacts and user data files.
+
+### Octane (experimental)
+
+YOLO supports Laravel Octane for turbocharged PHP applications.
+
+### Video Transcoding
+
+YOLO can provision resources on AWS to simplify video transcoding on AWS using AWS Elemental MediaConvert.
+
+### And Much More...
+
+- Least priviledge permissions with strong segregation across environments and apps
+- Seperate commands that run on deployment across worker groups
+- Scheduled MySQL backups using `mysqldump`
+- Control of build and deploy commands
+- Re-use existing VPCs, subnets, internet gateways and more
+
 ___
 
 ## Disclaimer
 
-YOLO is designed for PHP developers who want to manage AWS using an infrastructure-as-code approach, using plain-old PHP
-rather than CloudFormation / Terraform / K8s / Elastic Beanstalk / <some-other-fancy-alternative>.
+YOLO is designed for PHP developers who are comfortable managing AWS using an infrastructure-as-code approach.
 
-> [!IMPORTANT]
-> While YOLO has been battle-tested on apps serving millions of requests per day, it is not supposed to be a
-> set-and-forget solution for busy apps, but rather allows you to proactively manage, grow and adapt your infrastructure
-> as requirements change over time.
+It is, at it's core, a Symfony CLI app that leverages the AWS SDK, rather than CloudFormation / Terraform / K8s /
+Elastic
+Beanstalk / <some-other-fancy-alternative>.
+
+While YOLO has underpinned very large, mission-critical production applications, it is not intended to be a set and
+forget solution; rather it acts as a control plane that allows you to manage and expand your AWS footprint over time.
 
 It goes without saying, but use YOLO at your own risk.
 
@@ -173,7 +222,7 @@ environments:
       artefacts-bucket:
       cloudfront:
       alb:
-      transcoder: false
+      mediaconvert: false
       autoscaling:
         web:
         queue:

src/Aws.php

@@ -15,7 +15,6 @@
 use Aws\CloudWatch\CloudWatchClient;
 use Aws\CodeDeploy\CodeDeployClient;
 use Aws\AutoScaling\AutoScalingClient;
-use Aws\ElasticTranscoder\ElasticTranscoderClient;
 use Aws\ElasticLoadBalancingV2\ElasticLoadBalancingV2Client;
 
 class Aws
@@ -97,11 +96,6 @@ public static function elasticLoadBalancingV2(): ElasticLoadBalancingV2Client
         return Helpers::app('elasticLoadBalancingV2');
     }
 
-    public static function elasticTranscoder(): ElasticTranscoderClient
-    {
-        return Helpers::app('elasticTranscoder');
-    }
-
     public static function iam(): IamClient
     {
         return Helpers::app('iam');

src/AwsResources.php

@@ -13,7 +13,6 @@
 use Codinglabs\Yolo\Concerns\UsesCloudWatch;
 use Codinglabs\Yolo\Concerns\UsesCodeDeploy;
 use Codinglabs\Yolo\Concerns\UsesAutoscaling;
-use Codinglabs\Yolo\Concerns\UsesElasticTranscoder;
 use Codinglabs\Yolo\Concerns\UsesCertificateManager;
 use Codinglabs\Yolo\Concerns\UsesElasticLoadBalancingV2;
 
@@ -25,7 +24,6 @@ class AwsResources
     use UsesCodeDeploy;
     use UsesEc2;
     use UsesElasticLoadBalancingV2;
-    use UsesElasticTranscoder;
     use UsesIam;
     use UsesRds;
     use UsesRoute53;

src/Commands/DeployCommand.php

@@ -14,7 +14,7 @@
 class DeployCommand extends SteppedCommand
 {
     protected array $steps = [
-        Steps\Ensures\EnsureTranscoderExistsStep::class,
+        Steps\Ensures\EnsureIamRolesExistStep::class,
         Steps\Ensures\EnsureHostedZonesExistStep::class,
         Steps\Ensures\EnsureMultitenancyHostedZonesExistStep::class,
         Steps\Ensures\EnsureEnvIsConfiguredCorrectlyStep::class,

src/Commands/SyncComputeCommand.php

@@ -20,10 +20,6 @@ class SyncComputeCommand extends SteppedCommand
         //        // multitenancy
         //        Steps\Compute\SyncMultitenancyListenerOnPort443Step::class,
         //        Steps\Compute\AttachMultitenancySslCertificateToLoadBalancerListenerStep::class,
-
-        // transcoder
-        Steps\Compute\SyncElasticTranscoderPipelineStep::class,
-        Steps\Compute\SyncElasticTranscoderPresetStep::class,
     ];
 
     protected function configure(): void

src/Commands/SyncIamCommand.php

@@ -8,11 +8,13 @@
 class SyncIamCommand extends SteppedCommand
 {
     protected array $steps = [
-        Steps\Iam\SyncRoleStep::class,
-        Steps\Iam\SyncRolePolicyStep::class,
-        Steps\Iam\AttachRolePoliciesStep::class,
-        Steps\Iam\SyncInstanceProfileStep::class,
-        Steps\Iam\AttachRoleToInstanceProfileStep::class,
+        Steps\Iam\SyncEc2RoleStep::class,
+        Steps\Iam\SyncEc2RolePolicyStep::class,
+        Steps\Iam\AttachEc2RolePoliciesStep::class,
+        Steps\Iam\SyncEc2InstanceProfileStep::class,
+        Steps\Iam\AttachEc2RoleToInstanceProfileStep::class,
+        Steps\Iam\SyncMediaConvertRoleStep::class,
+        Steps\Iam\AttachMediaConvertRolePoliciesStep::class,
     ];
 
     protected function configure(): void

src/Commands/SyncNetworkCommand.php

@@ -32,7 +32,7 @@ class SyncNetworkCommand extends SteppedCommand
         Steps\Network\SyncRdsSecurityGroupStep::class,
 
         // sns
-        Steps\Network\SyncSnsTopicStep::class,
+        Steps\Network\SyncSnsAlarmTopicStep::class,
 
         // ssh
         Steps\Network\SyncKeyPairStep::class,

src/Concerns/RegistersAws.php

@@ -22,7 +22,7 @@
 use Codinglabs\Yolo\Enums\ServerGroup;
 use Aws\Credentials\CredentialProvider;
 use GuzzleHttp\Exception\ConnectException;
-use Aws\ElasticTranscoder\ElasticTranscoderClient;
+use Codinglabs\Yolo\Exceptions\IntegrityCheckException;
 use Aws\ElasticLoadBalancingV2\ElasticLoadBalancingV2Client;
 
 trait RegistersAws
@@ -43,7 +43,6 @@ protected function registerAwsServices(): void
         Helpers::app()->singleton('cloudWatch', fn () => new CloudWatchClient($arguments));
         Helpers::app()->singleton('ec2', fn () => new Ec2Client($arguments));
         Helpers::app()->singleton('elasticLoadBalancingV2', fn () => new ElasticLoadBalancingV2Client($arguments));
-        Helpers::app()->singleton('elasticTranscoder', fn () => new ElasticTranscoderClient($arguments));
         Helpers::app()->singleton('iam', fn () => new IamClient($arguments));
         Helpers::app()->singleton('rds', fn () => new RdsClient($arguments));
         Helpers::app()->singleton('route53', fn () => new Route53Client($arguments));
@@ -66,14 +65,20 @@ protected static function awsCredentials(): callable|array|null
             return null;
         }
 
-        // in CI (GitHub Actions) we use environment variables, otherwise we
-        // are using a local env value to point to the correct AWS profile.
-        return static::detectCiEnvironment()
-            ? [
+        // in CI (GitHub Actions) we use environment variables
+        if (static::detectCiEnvironment()) {
+            return [
                 'key' => env('AWS_ACCESS_KEY_ID'),
                 'secret' => env('AWS_SECRET_ACCESS_KEY'),
-            ]
-            : CredentialProvider::ini(Helpers::keyedEnv('AWS_PROFILE'));
+            ];
+        }
+
+        // otherwise we are using a local env value to point to the correct AWS profile.
+        if (in_array(Helpers::keyedEnv('AWS_PROFILE'), ['', null, 'default'])) {
+            throw new IntegrityCheckException(sprintf('Using the default AWS profile in your credentials file is risky. Name your profile to something specific and update %s in your .env file before proceeding.', Helpers::keyedEnvName('AWS_PROFILE')));
+        }
+
+        return CredentialProvider::ini(Helpers::keyedEnv('AWS_PROFILE'));
     }
 
     protected static function detectLocalEnvironment(): bool

src/Concerns/RunsSteppedCommands.php

@@ -71,7 +71,6 @@ protected function handleSteps(string $environment): int
 
                     // yellow
                     StepResult::SKIPPED => '<fg=yellow>SKIPPED</>',
-                    StepResult::WOULD_SKIP => '<fg=yellow>WOULD SKIP</>',
                     StepResult::CUSTOM_MANAGED => '<fg=yellow>CUSTOM MANAGED</>',
                     StepResult::WOULD_CREATE => '<fg=yellow>WOULD CREATE</>',
                     StepResult::WOULD_SYNC => '<fg=yellow>WOULD SYNC</>',
@@ -152,7 +151,7 @@ protected static function normaliseStep(Step $step, $pad = false, $bold = false,
                 ->when($bold && ! $step instanceof ExecutesTenantStep, fn (Stringable $string) => $string->wrap(before: '<options=bold>', after: '</>'))
         };
 
-        return $name->limit(50)
-            ->when($pad, fn (Stringable $string) => $string->padRight(50));
+        return $name->limit(70)
+            ->when($pad, fn (Stringable $string) => $string->padRight(70));
     }
 }