diff --git a/buildspec.yml b/buildspec.yml new file mode 100644 index 0000000000..8a8d64edbe --- /dev/null +++ b/buildspec.yml @@ -0,0 +1,19 @@ +version: 0.1 + +phases: + build: + commands: + - echo Build started on `date` + - mvn install --update-snapshots --threads 2.0C --batch-mode -DskipTests=true -Dmaven.javadoc.skip=true -Dmaven.compiler.showWarnings=false + - chmod -R a+x ./janusgraph-hbase-parent/janusgraph-hbase-098/bin && chmod -R a+x ./janusgraph-hbase-parent/janusgraph-hbase-10/bin + - mvn verify --projects $MODULE $ARGS + - mvn surefire-report:report --projects $MODULE + post_build: + commands: + - echo Build completed on `date` + - mkdir -p results/mvn + - echo "$MODULE" | tr "," "\n" | while read MOD; do if [ -d "${MOD}/target" ]; then mkdir -p results/mvn/${MOD} && cp -r ${MOD}/target results/mvn/${MOD}; fi; done +artifacts: + base-directory: results + files: + - mvn/**/* diff --git a/janusgraph-codepipelines-ci/README.md b/janusgraph-codepipelines-ci/README.md new file mode 100644 index 0000000000..008cd0e823 --- /dev/null +++ b/janusgraph-codepipelines-ci/README.md @@ -0,0 +1,146 @@ +# JanusGraph CodePipelines CI +CodePipelines CI is a mechanism JanusGraph can use to do release testing in massively +parallel fashion (to the extent of AWS CodePipelines and CodeBuild service limits). + +## Prerequisites +This procedure requires you to have an AWS account and a GitHub account. +It also requires you to create two service roles in IAM: one for CodePipeline and +one for CodeBuild. Finally, you need to have the [AWS CLI](https://aws.amazon.com/cli/) installed and on your path. + +1. Get a personal access token from [GitHub](https://github.com/settings/tokens) with `repo` and `admin:repo_hook` scopes. +The `repo` scope is used to push the latest updates on the branch selected below. The `admin:repo_hook` scope is for +setting up the post-commit hook on GitHub programmatically. +2. Navigate to the [AWS Console](https://console.aws.amazon.com) and +[create an IAM User](http://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html) with the following managed +policies: AmazonS3FullAccess, AWSCodePipelineFullAccess, AWSCodeBuildAdminAccess. This user should be created for +__Programmatic access__. +3. For this user, create security credentials and then register them in the `code-pipelines` profile on your computer +with `aws configure --profile code-pipelines`. Create this profile with a default region that +[supports CodePipeline and CodeBuild](https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/) +and a default [output format](http://docs.aws.amazon.com/cli/latest/userguide/controlling-output.html#controlling-output-format) +of your choice (`json`, `text`, and `table` are available). +4. Create an IAM policy for CodeBuild and associate it to a new service role as described +[here](http://docs.aws.amazon.com/codebuild/latest/userguide/setting-up.html#setting-up-service-role). Select the +__Amazon EC2__ role type as CodeBuild is not yet available on this page. Edit the new role's trust relationship +and replace it with the text in step 16 of the CodeBuild user guide. +5. Create an IAM policy for CodePipelines and associate it to a new service role as described +[here](http://docs.aws.amazon.com/codepipeline/latest/userguide/iam-identity-based-access-control.html#view-default-service-role-policy). +Select the __Amazon EC2__ role type as CodePipeline is not yet available on this page. Edit the new role's trust +relationship and replace the principal `ec2.amazonaws.com` with `codepipeline.amazonaws.com`. + +## Setup environment +Follow the steps below to prepare to create a test pipeline and run tests. First, clean up and set some constant +environment variables. + +```bash +mvn clean package +#constants +export AWS_PROFILE_NAME='code-pipelines' +export AWS_ACCOUNT_NUMBER=`aws sts get-caller-identity --profile ${AWS_PROFILE_NAME} --output text | cut -f1` +``` + +Second, configure the parameters below to customize your test stack. + +```bash +#GitHub personal access token you created in step 1 +export GITHUB_TOKEN='' +#GitHub organization or user name of the repository to build +export GITHUB_USERNAME='' +#GitHub repository name to use for builds +export GITHUB_REPOSITORY='' +#GitHub branch in the above repository to use for builds +export GITHUB_BRANCH='' +#Name of the AWS CodeBuild service role you created in step 4 +export AWS_CODEBUILD_ROLE='' +#Name of the AWS CodePipeline role you created in step 6 +export AWS_CODEPIPELINE_ROLE='' +#Region you want to set up your test stack in +export AWS_REGION_NAME='' +#Name of bucket you want to use or create and use for storing your build artifacts. +export AWS_S3_BUCKET_NAME='' +#Name of pipeline configuration file to use +export PIPELINE_CONFIGURATION='' +``` + +## Create test pipeline +This tool uses templates defined in YAML files to configure parallel builds in AWS CodePipeline. By default, +you can have up to [five parallel actions per stage](http://docs.aws.amazon.com/codepipeline/latest/userguide/limits.html) +in a pipeline and up to [twenty parallel builds](http://docs.aws.amazon.com/codebuild/latest/userguide/limits.html#limits-builds) +per region per account. If you need more, you can +[request to be whitelisted](http://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html) for more. + +Here is an example of a file that defines two pipelines with parallel build actions. + +```yaml +pipelines: +- name: j1 + parallelBuildActions: + - name: bdb + env: + - name: MODULE + value: janusgraph-berkeleyje + - name: h-l-s + env: + - name: MODULE + value: janusgraph-hadoop-parent/janusgraph-hadoop-2,janusgraph-lucene,janusgraph-solr + - name: cassandra + env: + - name: MODULE + value: janusgraph-cassandra + - name: test + env: + - name: MODULE + value: janusgraph-test +- name: j2 + parallelBuildActions: + - name: hbase098 + env: + - name: MODULE + value: janusgraph-hbase-parent/janusgraph-hbase-098 + - name: hbase10 + env: + - name: MODULE + value: janusgraph-hbase-parent/janusgraph-hbase-10 + - name: es + env: + - name: MODULE + value: janusgraph-es + - name: cql + env: + - name: MODULE + value: janusgraph-cql + +``` +The first pipeline is called `j1` and the second pipeline is called `j2`. Each of these pipelines have four +parallel build actions defined. Each build action is keyed at the action name and includes the environment +variables to be passed CodeBuild and used in the buildspec.yml file. + +To kick off the regular and TinkerPop tests. Use `export PIPELINE_CONFIGURATION=pipe.yml` for regular tests +and `export PIPELINE_CONFIGURATION=tp-pipe.yml` for TinkerPop tests. + +```bash +java -jar target/janusgraph-codepipelines-ci-0.2.0-SNAPSHOT.jar \ +--region ${AWS_REGION_NAME} \ +--bucket ${AWS_S3_BUCKET_NAME} \ +--github-owner ${GITHUB_USERNAME} \ +--github-repo ${GITHUB_REPOSITORY} \ +--github-branch ${GITHUB_BRANCH} \ +--profile ${AWS_PROFILE_NAME} \ +--codebuild-role-arn arn:aws:iam::${AWS_ACCOUNT_NUMBER}:role/${AWS_CODEBUILD_ROLE} \ +--codepipeline-role-arn arn:aws:iam::${AWS_ACCOUNT_NUMBER}:role/${AWS_CODEPIPELINE_ROLE} \ +--github-token ${GITHUB_TOKEN} \ +--pipelines ${PIPELINE_CONFIGURATION} +``` + +Navigate to the [AWS Console](https://console.aws.amazon.com/codepipeline) and check on the status of your pipeline to see status. + +## Cleaning up +After you follow the steps above, you will end up with resources in the following services: +1. Build artifacts and source code zips in S3 +2. CloudWatch logs +3. CodeBuild builds +4. CodePipeline pipelines, stages, and actions +5. IAM policies, roles and an IAM user +6. A GitHub personal access token + +You can delete these resources after you are finished running your tests. \ No newline at end of file diff --git a/janusgraph-codepipelines-ci/pipe.yml b/janusgraph-codepipelines-ci/pipe.yml new file mode 100644 index 0000000000..7f99705bc1 --- /dev/null +++ b/janusgraph-codepipelines-ci/pipe.yml @@ -0,0 +1,37 @@ +pipelines: +- name: j1 + parallelBuildActions: + - name: bdb + env: + - name: MODULE + value: janusgraph-berkeleyje + - name: h-l-s + env: + - name: MODULE + value: janusgraph-hadoop-parent/janusgraph-hadoop-2,janusgraph-lucene,janusgraph-solr + - name: cassandra + env: + - name: MODULE + value: janusgraph-cassandra + - name: test + env: + - name: MODULE + value: janusgraph-test +- name: j2 + parallelBuildActions: + - name: hbase098 + env: + - name: MODULE + value: janusgraph-hbase-parent/janusgraph-hbase-098 + - name: hbase10 + env: + - name: MODULE + value: janusgraph-hbase-parent/janusgraph-hbase-10 + - name: es + env: + - name: MODULE + value: janusgraph-es + - name: cql + env: + - name: MODULE + value: janusgraph-cql diff --git a/janusgraph-codepipelines-ci/pom.xml b/janusgraph-codepipelines-ci/pom.xml new file mode 100644 index 0000000000..201a28d932 --- /dev/null +++ b/janusgraph-codepipelines-ci/pom.xml @@ -0,0 +1,126 @@ + + + 4.0.0 + + org.janusgraph + janusgraph + 0.2.0-SNAPSHOT + ../pom.xml + + janusgraph-codepipelines-ci + JanusGraph CodePipelines CI: Distributed release testing. + http://janusgraph.org + + ${basedir}/.. + 2.0.0-preview-1 + 1.16.16 + + + + + software.amazon.awssdk + codepipeline + ${aws.sdk.version} + + + software.amazon.awssdk + codebuild + ${aws.sdk.version} + + + software.amazon.awssdk + s3 + ${aws.sdk.version} + + + software.amazon.awssdk + iam + ${aws.sdk.version} + + + com.google.guava + guava + + + commons-cli + commons-cli + + + org.projectlombok + lombok + ${lombok.version} + + + junit + junit + test + + + org.slf4j + slf4j-log4j12 + 1.7.12 + runtime + + + org.janusgraph + janusgraph-test + ${project.version} + test + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + 2.6.6 + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + 2.6.6 + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + package + + shade + + + + + org.janusgraph.codepipelines.AwsCodePipelinesCi + + + false + + + + + + + + + + joda-time + joda-time + 2.8.1 + + + io.netty + netty-handler + 4.1.9.Final + + + org.reactivestreams + reactive-streams + 1.0.0.final + + + + diff --git a/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/AwsCodePipelinesCi.java b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/AwsCodePipelinesCi.java new file mode 100644 index 0000000000..e422f0dd73 --- /dev/null +++ b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/AwsCodePipelinesCi.java @@ -0,0 +1,328 @@ +package org.janusgraph.codepipelines; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; + +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.assertj.core.util.Lists; +import org.janusgraph.codepipelines.model.ParallelBuildAction; +import org.janusgraph.codepipelines.model.PipelineDefinition; +import org.janusgraph.codepipelines.model.PipelineDefinitions; +import software.amazon.awssdk.auth.AwsCredentialsProvider; +import software.amazon.awssdk.auth.ProfileCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.codebuild.CodeBuildClient; +import software.amazon.awssdk.services.codebuild.model.ArtifactPackaging; +import software.amazon.awssdk.services.codebuild.model.ArtifactsType; +import software.amazon.awssdk.services.codebuild.model.ComputeType; +import software.amazon.awssdk.services.codebuild.model.CreateProjectRequest; +import software.amazon.awssdk.services.codebuild.model.EnvironmentType; +import software.amazon.awssdk.services.codebuild.model.EnvironmentVariable; +import software.amazon.awssdk.services.codebuild.model.ProjectArtifacts; +import software.amazon.awssdk.services.codebuild.model.ProjectEnvironment; +import software.amazon.awssdk.services.codebuild.model.ProjectSource; +import software.amazon.awssdk.services.codebuild.model.SourceType; +import software.amazon.awssdk.services.codepipeline.CodePipelineClient; +import software.amazon.awssdk.services.codepipeline.model.ActionCategory; +import software.amazon.awssdk.services.codepipeline.model.ActionDeclaration; +import software.amazon.awssdk.services.codepipeline.model.ActionOwner; +import software.amazon.awssdk.services.codepipeline.model.ActionTypeId; +import software.amazon.awssdk.services.codepipeline.model.ArtifactStore; +import software.amazon.awssdk.services.codepipeline.model.ArtifactStoreType; +import software.amazon.awssdk.services.codepipeline.model.CreatePipelineRequest; +import software.amazon.awssdk.services.codepipeline.model.CreatePipelineResponse; +import software.amazon.awssdk.services.codepipeline.model.InputArtifact; +import software.amazon.awssdk.services.codepipeline.model.OutputArtifact; +import software.amazon.awssdk.services.codepipeline.model.PipelineDeclaration; +import software.amazon.awssdk.services.codepipeline.model.StageDeclaration; +import software.amazon.awssdk.services.iam.model.IAMException; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.BucketLocationConstraint; +import software.amazon.awssdk.services.s3.model.CreateBucketConfiguration; +import software.amazon.awssdk.services.s3.model.CreateBucketRequest; +import software.amazon.awssdk.services.s3.model.PutBucketTaggingRequest; +import software.amazon.awssdk.services.s3.model.S3Exception; +import software.amazon.awssdk.services.s3.model.Tag; +import software.amazon.awssdk.services.s3.model.Tagging; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * This class orchestrates some CodePipelines stacks to run JanusGraph release tests in parallel. + * @author Alexander Patrikalakis + */ +@Slf4j +@ToString +public class AwsCodePipelinesCi { + + private static final Option REGION_OPTION = createRequiredOneArgOption("region", "AWS region to create build stack"); + private static final Option BUCKET_OPTION = createRequiredOneArgOption("bucket", "AWS S3 bucket to store artifacts"); + private static final Option CODEPIPELINE_ROLE_ARN_OPTION = + createRequiredOneArgOption("codepipeline-role-arn", "ARN of IAM role for CodePipeline to use"); + private static final Option GITHUB_OWNER_OPTION = + createRequiredOneArgOption("github-owner", "GitHub owner (organization) of the repository to build"); + private static final Option GITHUB_REPO_OPTION = createRequiredOneArgOption("github-repo", "GitHub repository to build"); + private static final Option GITHUB_BRANCH_OPTION = createRequiredOneArgOption("github-branch", "GitHub repository branch to build"); + private static final Option GITHUB_TOKEN_OPTION = + createRequiredOneArgOption("github-token", "GitHub personal access token for AWS CodeBuild to use to build"); + private static final Option PROFILE_OPTION = + createRequiredOneArgOption("profile", "AWS credential profile to use to create the stack"); + private static final Option PIPELINES_JSON_OPTION = + createRequiredOneArgOption("pipelines","Path to JSON file containing abbreviated pipeline definitions"); + private static final Option CODE_BUILD_SERVICE_ROLE_ARN_OPTION = + createRequiredOneArgOption("codebuild-role-arn", + "ARN of the service role for CodeBuild (http://docs.aws.amazon.com/codebuild/latest/userguide/setting-up.html#setting-up-service-role)"); + private static final Option CODE_BUILD_TIMEOUT_IN_MINUTES = + createOptionalOneArgOption("codebuild-timeout-in-minutes", + "Maximum time in minutes for a build. Must be greater than 0 and less than or equal to 480."); + private static final String DEFAULT_TIMEOUT_IN_MINUTES = "480"; + private static final Option CODE_BUILD_COMPUTE_IMAGE = createOptionalOneArgOption("codebuild-compute-image", + "Compute image to use for CodeBuild"); + private static final String DEFAULT_COMPUTE_IMAGE = "stephenreed/jenkins-java8-maven-git:latest"; + + private final List tags; + private final String s3Bucket; + private final String region; + private final String pipelineName; + private final String codeBuildServiceRoleArn; + private final String sourceOutputArtifactName; + private final S3Client s3; + private final CodePipelineClient codePipeline; + private final CodeBuildClient codeBuild; + private final String codePipelineRoleArn; + private final Map parallelBuildActions; + private final Map projectNameMap; + private final String githubToken; + private final String githubOwner; + private final String githubRepo; + private final String githubBranch; + private final String computeImage; + private final int timeoutInMinutes; + + private AwsCodePipelinesCi(final String region, final String s3bucket, PipelineDefinition definition, + final String codePipelineRoleArn, + final String githubToken, final String profileName, + final String githubOwner, final String githubRepo, + final String githubBranch, final String codeBuildServiceRoleArn, + final String computeImage, + final int timeoutInMinutes) { + this.s3Bucket = s3bucket; + this.tags = Lists.newArrayList(Tag.builder().key("project").value(definition.getName()).build(), + Tag.builder().key("date").value(Long.toString(System.currentTimeMillis())).build()); + this.region = region; + this.pipelineName = definition.getName(); + this.codeBuildServiceRoleArn = codeBuildServiceRoleArn; + this.sourceOutputArtifactName = pipelineName + "Source"; + this.parallelBuildActions = definition.getParallelBuildActions().stream().collect(Collectors.toMap(ParallelBuildAction::getName, Function.identity())); + this.projectNameMap = parallelBuildActions.keySet().stream().collect(Collectors.toMap(Function.identity(), envName -> pipelineName + "-" + envName)); + this.codePipelineRoleArn = codePipelineRoleArn; + this.githubToken = githubToken; + this.githubOwner = githubOwner; + this.githubRepo = githubRepo; + this.githubBranch = githubBranch; + this.computeImage = computeImage; + this.timeoutInMinutes = timeoutInMinutes; + + //setup clients + final Region regionEnum = Region.of(region); + final AwsCredentialsProvider provider = ProfileCredentialsProvider.builder().profileName(profileName).build(); + this.s3 = S3Client.builder().region(regionEnum).credentialsProvider(provider).build(); + this.codePipeline = CodePipelineClient.builder().region(regionEnum).credentialsProvider(provider).build(); + this.codeBuild = CodeBuildClient.builder().region(regionEnum).credentialsProvider(provider).build(); + } + + private String getBuildOutputArtifactName(String action) { + return this.projectNameMap.get(action) + "-artifacts"; + } + + private ActionDeclaration createBuildActionDeclaration(ParallelBuildAction action, int version) { + final String actionName = action.getName(); + return ActionDeclaration.builder() + .name(actionName) + .inputArtifacts(InputArtifact.builder().name(sourceOutputArtifactName).build()) + .actionTypeId(ActionTypeId.builder() + .category(ActionCategory.Build) + .owner(ActionOwner.AWS) + .provider("CodeBuild") + .version(Integer.toString(version)).build() + ) + .outputArtifacts(OutputArtifact.builder().name(getBuildOutputArtifactName(actionName)).build()) + .configuration(ImmutableMap.of("ProjectName", projectNameMap.get(actionName))) + .runOrder(Integer.valueOf(1)).build(); //all the builds will run in parallel + } + + private StageDeclaration createBuildStageDeclaration(int version) { + return StageDeclaration.builder() + .name("Build") + .actions(parallelBuildActions.values().stream() + .map(action -> createBuildActionDeclaration(action, version)) + .collect(Collectors.toList())).build(); + } + + private StageDeclaration createSourceStageDeclaration(int version) { + Preconditions.checkArgument(version > 0); + return StageDeclaration.builder().name("Source") + .actions(ActionDeclaration.builder().name(pipelineName) + .actionTypeId(ActionTypeId.builder() + .category(ActionCategory.Source) + .owner(ActionOwner.ThirdParty) + .provider("GitHub") + .version(Integer.toString(version)).build()) + .outputArtifacts(OutputArtifact.builder().name(sourceOutputArtifactName).build()) + .configuration(ImmutableMap.of( + "Owner", githubOwner, + "Repo", githubRepo, + "Branch", githubBranch, + "OAuthToken", githubToken)) + .runOrder(1) + .build() + ) + .build(); + } + + private CreateProjectRequest createCodeBuildProjectRequest(ParallelBuildAction action, + String serviceRoleArn, + String artifactsName, + ComputeType computeType, + String computeImage, + int timeoutInMinutes) { + log.info("Creating CodeBuild project for " + action.getName()); + Preconditions.checkArgument(timeoutInMinutes <= 480 && timeoutInMinutes > 0, + "timeoutInMinutes must be greater than zero and less than or equal to 8 hours"); + return CreateProjectRequest.builder() + .name(projectNameMap.get(action.getName())) + .serviceRole(serviceRoleArn) + //.withTags(null) //TODO fix + .artifacts(ProjectArtifacts.builder().packaging(ArtifactPackaging.NONE).type(ArtifactsType.CODEPIPELINE).name(artifactsName).build()) + .timeoutInMinutes(timeoutInMinutes) + .environment(ProjectEnvironment.builder() + .computeType(computeType) + .image(computeImage) + .type(EnvironmentType.LINUX_CONTAINER) + .privilegedMode(Boolean.TRUE) + .environmentVariables(action.getEnv().stream() + .map(e -> EnvironmentVariable.builder().name(e.getName()).value(e.getValue()).build()) + .collect(Collectors.toList())).build()) + .source(ProjectSource.builder().type(SourceType.CODEPIPELINE).build()).build(); + } + + private void run() { + try { + s3.createBucket(CreateBucketRequest.builder() + .createBucketConfiguration(CreateBucketConfiguration.builder().locationConstraint(BucketLocationConstraint.fromValue(region)).build()) + .bucket(s3Bucket) + .build()); + s3.putBucketTagging(PutBucketTaggingRequest.builder().bucket(s3Bucket).tagging(Tagging.builder().tagSet(tags).build()).build()); + log.info("Created bucket " + s3Bucket + " in region " + region); + } catch (S3Exception e) { + if (e.getErrorCode().equals("BucketAlreadyOwnedByYou")) { + log.info("Bucket " + s3Bucket + " in region " + region + " already owned by you"); + } else { + throw new IllegalArgumentException("Unable to create/configure bucket", e); + } + } + + parallelBuildActions.entrySet().stream() + .map(entry -> createCodeBuildProjectRequest(entry.getValue(), + codeBuildServiceRoleArn, + getBuildOutputArtifactName(entry.getKey()), + ComputeType.BUILD_GENERAL1_LARGE, //TODO externalize + computeImage, + timeoutInMinutes) + ) + .peek(project -> log.info(project.toString())) + .forEach(codeBuild::createProject); + + final int version = 1; + final StageDeclaration sourceStage = createSourceStageDeclaration(version); + final StageDeclaration buildStage = createBuildStageDeclaration(version); + final CreatePipelineResponse pipeline = codePipeline.createPipeline( + CreatePipelineRequest.builder().pipeline(PipelineDeclaration.builder() + .roleArn(codePipelineRoleArn) + .name(pipelineName) + .version(Integer.MAX_VALUE) + .stages(sourceStage, buildStage) + .artifactStore(ArtifactStore.builder().type(ArtifactStoreType.S3).location(s3Bucket).build()).build()).build()); + + Preconditions.checkNotNull(pipeline, "created pipeline was null"); + } + + private static Option createRequiredOneArgOption(final String longOpt, final String description) { + final Option o = new Option(null /*shortOpt*/, longOpt, true /*hasArg*/, description); + o.setRequired(true); + o.setArgs(1); + return o; + } + + private static Option createOptionalOneArgOption(final String longOpt, final String description) { + final Option o = new Option(null /*shortOpt*/, longOpt, true /*hasArg*/, description); + o.setRequired(false); + o.setArgs(1); + return o; + } + + public static void main(String... args) { + int status = 0; + try { + final Options options = new Options(); + options.addOption(REGION_OPTION); + options.addOption(BUCKET_OPTION); + options.addOption(CODEPIPELINE_ROLE_ARN_OPTION); + options.addOption(GITHUB_OWNER_OPTION); + options.addOption(GITHUB_REPO_OPTION); + options.addOption(GITHUB_BRANCH_OPTION); + options.addOption(GITHUB_TOKEN_OPTION); + options.addOption(PROFILE_OPTION); + options.addOption(PIPELINES_JSON_OPTION); + options.addOption(CODE_BUILD_SERVICE_ROLE_ARN_OPTION); + options.addOption(CODE_BUILD_COMPUTE_IMAGE); + options.addOption(CODE_BUILD_TIMEOUT_IN_MINUTES); + final CommandLineParser parser = new DefaultParser(); + final CommandLine cmd = parser.parse(options, args); + + //TODO ConstructorProperties was introduced in jacskon 2.7, use it + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + final PipelineDefinitions definitions = mapper.readValue(new File(cmd.getOptionValue(PIPELINES_JSON_OPTION.getLongOpt())), PipelineDefinitions.class); + definitions.getPipelines().forEach(definition -> { + final AwsCodePipelinesCi ci = new AwsCodePipelinesCi( + cmd.getOptionValue(REGION_OPTION.getLongOpt()), + cmd.getOptionValue(BUCKET_OPTION.getLongOpt()), + definition, + cmd.getOptionValue(CODEPIPELINE_ROLE_ARN_OPTION.getLongOpt()), + cmd.getOptionValue(GITHUB_TOKEN_OPTION.getLongOpt()), + cmd.getOptionValue(PROFILE_OPTION.getLongOpt()), + cmd.getOptionValue(GITHUB_OWNER_OPTION.getLongOpt()), + cmd.getOptionValue(GITHUB_REPO_OPTION.getLongOpt()), + cmd.getOptionValue(GITHUB_BRANCH_OPTION.getLongOpt()), + cmd.getOptionValue(CODE_BUILD_SERVICE_ROLE_ARN_OPTION.getLongOpt()), + Optional.ofNullable(cmd.getOptionValue(CODE_BUILD_COMPUTE_IMAGE.getLongOpt())).orElse(DEFAULT_COMPUTE_IMAGE), + Integer.valueOf(Optional.ofNullable(cmd.getOptionValue(CODE_BUILD_TIMEOUT_IN_MINUTES.getLongOpt())).orElse(DEFAULT_TIMEOUT_IN_MINUTES))); + ci.run(); + }); + } catch (ParseException | IllegalArgumentException e) { + log.error(e.getMessage(), e); + status = 22; //EINVAL + } catch (IAMException e) { + log.error(e.getMessage(), e); + status = 1; //EPERM + } catch (Exception e) { + log.error(e.getMessage(), e); + status = 11; //EAGAIN + } + System.exit(status); + } +} diff --git a/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/EnvironmentMapping.java b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/EnvironmentMapping.java new file mode 100644 index 0000000000..3be51a6d7c --- /dev/null +++ b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/EnvironmentMapping.java @@ -0,0 +1,19 @@ +package org.janusgraph.codepipelines.model; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Value; + +import java.util.List; + +@Value +public class EnvironmentMapping { + private final String name; + private final String value; + @JsonCreator + public EnvironmentMapping(@JsonProperty("name") String name, @JsonProperty("value") String value) { + this.name = name; + this.value = value; + } +} diff --git a/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/ParallelBuildAction.java b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/ParallelBuildAction.java new file mode 100644 index 0000000000..fffb9e7086 --- /dev/null +++ b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/ParallelBuildAction.java @@ -0,0 +1,19 @@ +package org.janusgraph.codepipelines.model; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Value; + +@Value +public class ParallelBuildAction { + private final String name; + private final List env; + @JsonCreator + public ParallelBuildAction(@JsonProperty("name") String name, @JsonProperty("env") List env) { + this.name = name; + this.env = env; + } +} diff --git a/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/PipelineDefinition.java b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/PipelineDefinition.java new file mode 100644 index 0000000000..3689f016ea --- /dev/null +++ b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/PipelineDefinition.java @@ -0,0 +1,19 @@ +package org.janusgraph.codepipelines.model; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Value; + +@Value +public class PipelineDefinition { + private final String name; + private final List parallelBuildActions; + @JsonCreator + public PipelineDefinition(@JsonProperty("name") String name, @JsonProperty("parallelBuildActions") List parallelBuildActions) { + this.name = name; + this.parallelBuildActions = parallelBuildActions; + } +} diff --git a/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/PipelineDefinitions.java b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/PipelineDefinitions.java new file mode 100644 index 0000000000..c6027d014e --- /dev/null +++ b/janusgraph-codepipelines-ci/src/main/java/org/janusgraph/codepipelines/model/PipelineDefinitions.java @@ -0,0 +1,17 @@ +package org.janusgraph.codepipelines.model; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonCreator; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Value; + +@Value +public class PipelineDefinitions { + private final List pipelines; + @JsonCreator + public PipelineDefinitions(@JsonProperty("pipelines") List pipelines) { + this.pipelines = pipelines; + } +} diff --git a/janusgraph-codepipelines-ci/src/main/resources/log4j.properties b/janusgraph-codepipelines-ci/src/main/resources/log4j.properties new file mode 100644 index 0000000000..9b72c91828 --- /dev/null +++ b/janusgraph-codepipelines-ci/src/main/resources/log4j.properties @@ -0,0 +1,13 @@ +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + +# Set root logger level to the designated level and its only appender to A1. +log4j.rootLogger=INFO, A1 + +log4j.logger.org.apache.cassandra=INFO +log4j.logger.org.apache.hadoop=INFO +log4j.logger.org.apache.zookeeper=INFO diff --git a/janusgraph-codepipelines-ci/src/test/java/org/janusgraph/codepipelines/TestContainerOne.java b/janusgraph-codepipelines-ci/src/test/java/org/janusgraph/codepipelines/TestContainerOne.java new file mode 100644 index 0000000000..26d3254b04 --- /dev/null +++ b/janusgraph-codepipelines-ci/src/test/java/org/janusgraph/codepipelines/TestContainerOne.java @@ -0,0 +1,29 @@ +package org.janusgraph.codepipelines; + +import org.junit.Test; + +/** + * @author Alexander Patrikalakis + */ +public class TestContainerOne { + public static class TraversalAlpha { + @Test + public void testOne() { + //noop + } + @Test + public void testTwo() { + //noop + } + } + public static class TraversalBeta { + @Test + public void testOne() { + //noop + } + @Test + public void testTwo() { + //noop + } + } +} diff --git a/janusgraph-codepipelines-ci/src/test/java/org/janusgraph/codepipelines/TestContainerTwo.java b/janusgraph-codepipelines-ci/src/test/java/org/janusgraph/codepipelines/TestContainerTwo.java new file mode 100644 index 0000000000..d7e34d6cd7 --- /dev/null +++ b/janusgraph-codepipelines-ci/src/test/java/org/janusgraph/codepipelines/TestContainerTwo.java @@ -0,0 +1,29 @@ +package org.janusgraph.codepipelines; + +import org.junit.Test; + +/** + * @author Alexander Patrikalakis + */ +public class TestContainerTwo { + public static class TraversalAlpha { + @Test + public void testOne() { + //noop + } + @Test + public void testTwo() { + //noop + } + } + public static class TraversalBeta { + @Test + public void testOne() { + //noop + } + @Test + public void testTwo() { + //noop + } + } +} diff --git a/janusgraph-codepipelines-ci/src/test/resources/excludes b/janusgraph-codepipelines-ci/src/test/resources/excludes new file mode 100644 index 0000000000..dad47c1e45 --- /dev/null +++ b/janusgraph-codepipelines-ci/src/test/resources/excludes @@ -0,0 +1 @@ +**/*.java* diff --git a/janusgraph-codepipelines-ci/src/test/resources/longTests1 b/janusgraph-codepipelines-ci/src/test/resources/longTests1 new file mode 100644 index 0000000000..3bce42eaca --- /dev/null +++ b/janusgraph-codepipelines-ci/src/test/resources/longTests1 @@ -0,0 +1,2 @@ +**/*TestContainerTwo$TraversalBeta* +**/*TestContainerTwo$TraversalAlpha* diff --git a/janusgraph-codepipelines-ci/src/test/resources/longTests2 b/janusgraph-codepipelines-ci/src/test/resources/longTests2 new file mode 100644 index 0000000000..abf80d1ecf --- /dev/null +++ b/janusgraph-codepipelines-ci/src/test/resources/longTests2 @@ -0,0 +1 @@ +**/*TestContainerTwo$TraversalAlpha* \ No newline at end of file diff --git a/janusgraph-codepipelines-ci/tp-pipe.yml b/janusgraph-codepipelines-ci/tp-pipe.yml new file mode 100644 index 0000000000..91971d1866 --- /dev/null +++ b/janusgraph-codepipelines-ci/tp-pipe.yml @@ -0,0 +1,47 @@ +pipelines: +- name: jtp1 + parallelBuildActions: + - name: e-h-l-s-tp + env: + - name: MODULE + value: janusgraph-es,janusgraph-hadoop-parent/janusgraph-hadoop-2,janusgraph-lucene,janusgraph-solr + - name: ARGS + value: "-Dtest.skip.tp=false -DskipTests=true" + - name: bdb-tp + env: + - name: MODULE + value: janusgraph-berkeleyje + - name: ARGS + value: "-Dtest.skip.tp=false -DskipTests=true" +- name: jtp2 + parallelBuildActions: + - name: cassandra-tp + env: + - name: MODULE + value: janusgraph-cassandra + - name: ARGS + value: "-Dtest.skip.tp=false -DskipTests=true" + - name: test-tp + env: + - name: MODULE + value: janusgraph-test + - name: ARGS + value: "-Dtest.skip.tp=false -DskipTests=true" + - name: hbase098-tp + env: + - name: MODULE + value: janusgraph-hbase-parent/janusgraph-hbase-098 + - name: ARGS + value: "-Dtest.skip.tp=false -DskipTests=true" + - name: hbase10-tp + env: + - name: MODULE + value: janusgraph-hbase-parent/janusgraph-hbase-10 + - name: ARGS + value: "-Dtest.skip.tp=false -DskipTests=true" + - name: cql-tp + env: + - name: MODULE + value: janusgraph-cql + - name: ARGS + value: "-Dtest.skip.tp=false -DskipTests=true" diff --git a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStorageSetup.java b/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStorageSetup.java index 5c5745c63b..1b4765603f 100644 --- a/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStorageSetup.java +++ b/janusgraph-hbase-parent/janusgraph-hbase-core/src/test/java/org/janusgraph/HBaseStorageSetup.java @@ -132,7 +132,7 @@ public synchronized static HBaseStatus startHBase() throws IOException { log.info("Starting HBase"); String scriptPath = getScriptDirForHBaseVersion(HBASE_TARGET_VERSION) + "/hbase-daemon.sh"; - runCommand(scriptPath, "--config", getConfDirForHBaseVersion(HBASE_TARGET_VERSION), "start", "master"); + DaemonRunner.runCommand(scriptPath, "--config", getConfDirForHBaseVersion(HBASE_TARGET_VERSION), "start", "master"); HBASE = HBaseStatus.write(HBASE_STAT_FILE, HBASE_TARGET_VERSION); @@ -229,7 +229,7 @@ private synchronized static void shutdownHBase(HBaseStatus stat) { log.info("Shutting down HBase..."); // First try graceful shutdown through the script... - runCommand(stat.getScriptDir() + "/hbase-daemon.sh", "--config", stat.getConfDir(), "stop", "master"); + DaemonRunner.runCommand(stat.getScriptDir() + "/hbase-daemon.sh", "--config", stat.getConfDir(), "stop", "master"); log.info("Shutdown HBase"); @@ -239,82 +239,4 @@ private synchronized static void shutdownHBase(HBaseStatus stat) { HBASE = null; } - - /** - * Run the parameter as an external process. Returns if the command starts - * without throwing an exception and returns exit status 0. Throws an - * exception if there's any problem invoking the command or if it does not - * return zero exit status. - * - * Blocks indefinitely while waiting for the command to complete. - * - * @param argv - * passed directly to {@link ProcessBuilder}'s constructor - */ - private static void runCommand(String... argv) { - - final String cmd = Joiner.on(" ").join(argv); - log.info("Executing {}", cmd); - - ProcessBuilder pb = new ProcessBuilder(argv); - pb.redirectErrorStream(true); - Process startup; - try { - startup = pb.start(); - } catch (IOException e) { - throw new RuntimeException(e); - } - StreamLogger sl = new StreamLogger(startup.getInputStream()); - sl.setDaemon(true); - sl.start(); - - try { - int exitcode = startup.waitFor(); // wait for script to return - if (0 == exitcode) { - log.info("Command \"{}\" exited with status 0", cmd); - } else { - throw new RuntimeException("Command \"" + cmd + "\" exited with status " + exitcode); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - try { - sl.join(1000L); - } catch (InterruptedException e) { - log.warn("Failed to cleanup stdin handler thread after running command \"{}\"", cmd, e); - } - } - - /* - * This could be retired in favor of ProcessBuilder.Redirect when we move to - * source level 1.7. - */ - private static class StreamLogger extends Thread { - - private final BufferedReader reader; - private static final Logger log = - LoggerFactory.getLogger(StreamLogger.class); - - private StreamLogger(InputStream is) { - this.reader = new BufferedReader(new InputStreamReader(is)); - } - - @Override - public void run() { - String line; - try { - while (null != (line = reader.readLine())) { - log.info("> {}", line); - if (Thread.currentThread().isInterrupted()) { - break; - } - } - - log.info("End of stream."); - } catch (IOException e) { - log.error("Unexpected IOException while reading stream {}", reader, e); - } - } - } } diff --git a/janusgraph-test/src/main/java/org/janusgraph/DaemonRunner.java b/janusgraph-test/src/main/java/org/janusgraph/DaemonRunner.java index b781cdb539..b39fd9d6e9 100644 --- a/janusgraph-test/src/main/java/org/janusgraph/DaemonRunner.java +++ b/janusgraph-test/src/main/java/org/janusgraph/DaemonRunner.java @@ -132,7 +132,7 @@ private synchronized void killAndUnregisterHook(final S stat) { * @param argv * passed directly to {@link ProcessBuilder}'s constructor */ - protected static void runCommand(String... argv) { + public static void runCommand(String... argv) { final String cmd = Joiner.on(" ").join(argv); log.info("Executing {}", cmd); @@ -143,12 +143,22 @@ protected static void runCommand(String... argv) { try { startup = pb.start(); } catch (IOException e) { - throw new RuntimeException(e); + throw new RuntimeException("Unable to start process in " + System.getProperty("user.dir") + ": " + e.getMessage(), e); } StreamLogger sl = new StreamLogger(startup.getInputStream()); sl.setDaemon(true); sl.start(); + waitForProcessAndCheckStatus(cmd, startup); + + try { + sl.join(1000L); + } catch (InterruptedException e) { + log.warn("Failed to cleanup stdin handler thread after running command \"{}\"", cmd, e); + } + } + + private static void waitForProcessAndCheckStatus(String cmd, Process startup) { try { int exitcode = startup.waitFor(); // wait for script to return if (0 == exitcode) { @@ -159,12 +169,6 @@ protected static void runCommand(String... argv) { } catch (InterruptedException e) { throw new RuntimeException(e); } - - try { - sl.join(1000L); - } catch (InterruptedException e) { - log.warn("Failed to cleanup stdin handler thread after running command \"{}\"", cmd, e); - } } /* diff --git a/pom.xml b/pom.xml index 50bc19ee5a..d2207e35b0 100644 --- a/pom.xml +++ b/pom.xml @@ -118,6 +118,7 @@ false + janusgraph-codepipelines-ci janusgraph-core janusgraph-test janusgraph-berkeleyje @@ -308,9 +309,6 @@ **/*StructureTest.java alphabetical - ${test.skip.tp} ${project.build.directory} @@ -318,7 +316,7 @@ true - +