diff --git a/.ackrc b/.ackrc deleted file mode 100644 index 58be49352..000000000 --- a/.ackrc +++ /dev/null @@ -1,3 +0,0 @@ ---ignore-dir=build/work ---ignore-dir=src/api-umbrella/web-app/public/web-assets ---ignore-dir=website/source/javascripts/_vendor diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 000000000..7945bf0a2 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,45 @@ +version: 2 +jobs: + build: + parallelism: 3 + working_directory: /app + docker: + - image: circleci/buildpack-deps:18.04-browsers + user: root + steps: + - checkout + # Install any system package dependencies. + - run: env INSTALL_TEST_DEPENDENCIES=true ./tasks/install-system-build-dependencies + - restore_cache: + # CircleCI's prefix based matching will mean the cache from the last + # cache for this branch will be restored. + key: cache-build-deps-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ arch }}- + # The bootstrap's "task" file will be cached on subsequent runs, but due + # to the reliance on timestamps, this fact might not get picked up in the + # CI environment. So if a stamp file exists for the initial bootstrap + # process, re-touch it to prevent re-downloads. + - run: touch -c build/work/stamp/bootstrap-* + # Build all the API Umbrella software dependencies. + - run: ./configure + - run: make all test-deps + - run: make clean:dev + # Cache the staged build data and task checksums of what's been + # completed. + - save_cache: + key: cache-build-deps-{{ .Environment.CACHE_VERSION }}-{{ .Branch }}-{{ arch }}-{{ epoch }} + paths: + - .task + - build/work/dev-env + - build/work/stage + - build/work/stamp + - build/work/task + - build/work/tasks + - build/work/test-env + - run: groupadd -r api-umbrella && useradd -r -g api-umbrella -s /sbin/nologin -d /opt/api-umbrella -c "API Umbrella user" api-umbrella + - run: make test:circle-ci + - store_test_results: + path: test/tmp/reports + - store_artifacts: + path: test/tmp/run/api-umbrella-root/var/log + - store_artifacts: + path: test/tmp/capybara diff --git a/.dockerignore b/.dockerignore index d61674ac1..4c930b63a 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,14 +1,7 @@ -# CMake -CMakeCache.txt -CMakeFiles -Makefile -cmake_install.cmake -install_manifest.txt -install_manifest_core.txt -install_manifest_hadoop-analytics.txt - +.task build/work build/package/work +build/package/verify/.rnd .vagrant workspace @@ -16,16 +9,11 @@ chef/cookbooks chef/tmp .bundle .ruby-version -CMakeCache.txt -CMakeFiles +Dockerfile* Makefile -cmake_install.cmake +docker-compose.yml install_manifest.txt install_manifest_core.txt -install_manifest_hadoop-analytics.txt -build/work -build/package/work -build/package/verify/.rnd deploy/.env deploy/log deploy/tmp diff --git a/.gitignore b/.gitignore index 669afaf3f..9c3f81cb9 100644 --- a/.gitignore +++ b/.gitignore @@ -17,18 +17,13 @@ # Ignore bundler config /.bundle -# Ignore Circle CI items, so the GitHub pages publish task can work. +# Ignore Circle CI items, so the GitHub pages publish task can work. /.ruby-version -# CMake -/CMakeCache.txt -/CMakeFiles -/Makefile -/cmake_install.cmake -/install_manifest.txt -/install_manifest_core.txt -/install_manifest_hadoop-analytics.txt +# Build task checksums +/.task +/Makefile /build/work /build/package/work /build/package/verify/.rnd diff --git a/.luacheckrc b/.luacheckrc index df3a8249a..e699bc39e 100644 --- a/.luacheckrc +++ b/.luacheckrc @@ -1,12 +1,28 @@ std = "ngx_lua" globals = { - "DEBUG", "WORKER_GROUP_ID", - "config", - "elasticsearch_templates", - "inspect", - "user_agent_parser_data", } max_line_length = false + +files["src/api-umbrella/auto-ssl"] = { + globals = { + "auto_ssl", + }, +} + +files["templates/etc/trafficserver"] = { + std = "luajit", + globals = { + "TS_LUA_CACHE_LOOKUP_HIT_FRESH", + "TS_LUA_CACHE_LOOKUP_HIT_STALE", + "TS_LUA_REMAP_DID_REMAP", + "do_global_read_request", + "do_global_read_response", + "do_global_send_request", + "do_global_send_response", + "do_remap", + "ts", + }, +} diff --git a/.rubocop.yml b/.rubocop.yml index c2559ab81..2b09cb2b8 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,13 +1,51 @@ AllCops: - TargetRubyVersion: 2.3 + TargetRubyVersion: 2.4 Exclude: - bin/**/* + - scripts/**/* + - src/api-umbrella/web-app/bin/**/* + - src/api-umbrella/web-app/tmp/**/* + - src/api-umbrella/web-app/vendor/**/* - tmp/**/* - vendor/**/* Rails: Enabled: true +Bundler/OrderedGems: + Enabled: false + +Layout/AlignHash: + # https://github.com/rubocop-hq/rubocop/issues/6291 + EnforcedLastArgumentHashStyle: ignore_implicit + +Layout/AlignParameters: + EnforcedStyle: with_fixed_indentation + +Layout/CaseIndentation: + EnforcedStyle: end + +Layout/EndAlignment: + EnforcedStyleAlignWith: start_of_line + +Layout/FirstParameterIndentation: + EnforcedStyle: consistent + +Layout/IndentArray: + EnforcedStyle: consistent + +Layout/IndentHash: + EnforcedStyle: consistent + +Layout/MultilineMethodCallIndentation: + EnforcedStyle: indented + +Layout/MultilineOperationIndentation: + EnforcedStyle: indented + +Layout/SpaceAroundKeyword: + Enabled: false + Lint/AssignmentInCondition: AllowSafeAssignment: false @@ -53,22 +91,10 @@ Naming/HeredocDelimiterNaming: Performance/RedundantMerge: Enabled: false -Layout/AlignHash: - Enabled: false - -Layout/AlignParameters: - Enabled: false - -Layout/EmptyLinesAroundArguments: +Rails/FindBy: Enabled: false -Layout/IndentArray: - EnforcedStyle: consistent - -Layout/IndentHash: - EnforcedStyle: consistent - -Layout/SpaceAroundKeyword: +Rails/HasAndBelongsToMany: Enabled: false Style/AsciiComments: @@ -81,11 +107,11 @@ Style/ClassAndModuleChildren: Enabled: false Style/ClassCheck: - Enabled: false + EnforcedStyle: kind_of? Style/CollectionMethods: PreferredMethods: - find: 'detect' + find: detect Style/ConditionalAssignment: Enabled: false @@ -93,18 +119,12 @@ Style/ConditionalAssignment: Style/Documentation: Enabled: false -Style/EachWithObject: - Enabled: false - Style/EmptyElse: EnforcedStyle: empty Style/EmptyMethod: Enabled: false -Style/Encoding: - Enabled: false - Style/FrozenStringLiteralComment: Enabled: false @@ -133,7 +153,7 @@ Style/NumericLiterals: Enabled: false Style/NumericPredicate: - Enabled: false + EnforcedStyle: comparison Style/OneLineConditional: Enabled: false @@ -163,10 +183,10 @@ Style/SignalException: Enabled: false Style/StringLiterals: - Enabled: false + EnforcedStyle: double_quotes Style/StringLiteralsInInterpolation: - Enabled: false + EnforcedStyle: double_quotes Style/SymbolArray: Enabled: false @@ -187,17 +207,4 @@ Style/WhileUntilModifier: Enabled: false Style/WordArray: - Enabled: false - -Rails/Delegate: - Enabled: false - -Rails/FindBy: - Enabled: false - -Rails/HasAndBelongsToMany: - Enabled: false - -Rails/Output: - Exclude: - - db/migrate/** + EnforcedStyle: brackets diff --git a/Berksfile b/Berksfile deleted file mode 100644 index e87dbfac9..000000000 --- a/Berksfile +++ /dev/null @@ -1,3 +0,0 @@ -source "https://supermarket.chef.io" - -cookbook "api-umbrella", :git => "https://github.com/NREL/api-umbrella-cookbook.git" diff --git a/Berksfile.lock b/Berksfile.lock deleted file mode 100644 index a57aeabac..000000000 --- a/Berksfile.lock +++ /dev/null @@ -1,28 +0,0 @@ -DEPENDENCIES - api-umbrella - git: https://github.com/NREL/api-umbrella-cookbook.git - revision: 5c6b64bf31780b8e6a933ba00b32305a4a3f8960 - -GRAPH - api-umbrella (0.7.1) - build-essential (>= 0.0.0) - sudo (>= 0.0.0) - ulimit (>= 0.0.0) - yum (>= 0.0.0) - yum-epel (>= 0.0.0) - build-essential (8.0.0) - mingw (>= 1.1) - seven_zip (>= 0.0.0) - compat_resource (12.16.3) - mingw (2.0.0) - seven_zip (>= 0.0.0) - ohai (5.0.3) - seven_zip (2.0.2) - windows (>= 1.2.2) - sudo (3.3.1) - ulimit (0.4.0) - windows (3.0.5) - ohai (>= 4.0.0) - yum (5.0.1) - yum-epel (2.1.1) - compat_resource (>= 12.16.3) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cda6a341..b5b5ae3b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Fixed - **Fix URL handling for query strings containing "api\_key":** It was possible that API Umbrella was stripping the string "api\_key" from inside URLs before passing requests to the API backend in some unexpected cases. The `api_key` query parameter should still be stripped, but other instances of "api\_key" elsewhere in the URL (for example as a value, like `?foo=api_key`), are now retained. +- **Fix redirect rewriting when operating on custom ports:** If API Umbrella was running on custom HTTP or HTTP ports, redirects from API backends may not have been to the correct port. ## 0.14.4 (2017-07-15) diff --git a/CMakeLists.txt b/CMakeLists.txt deleted file mode 100644 index 911f9503c..000000000 --- a/CMakeLists.txt +++ /dev/null @@ -1,137 +0,0 @@ -cmake_minimum_required(VERSION 3.6.0 FATAL_ERROR) -project(api-umbrella) - -option(ENABLE_HADOOP_ANALYTICS "Build dependencies for Hadoop analytics" off) -option(ENABLE_TEST_DEPENDENCIES "Build dependencies for running tests" off) -option(ENABLE_DEPLOY_ONLY "Only build dependencies for a deployment overlaying an existing package install" off) - -# Installation prefix -if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set(CMAKE_INSTALL_PREFIX /opt/api-umbrella CACHE PATH "default install path" FORCE) -endif() -set(INSTALL_PREFIX_EMBEDDED ${CMAKE_INSTALL_PREFIX}/embedded) - -# Where to install cmake ExternalProjects. -set(EP_BASE build/work/external-projects) -set_directory_properties(PROPERTIES EP_BASE ${EP_BASE}) -set(WORK_DIR ${CMAKE_BINARY_DIR}/build/work) -set(PACKAGE_WORK_DIR ${CMAKE_BINARY_DIR}/build/package/work) - -# Directory to store "stamp" files indicating when a step was completed or last -# touched. -set(STAMP_DIR ${WORK_DIR}/stamp) -execute_process(COMMAND mkdir -p ${STAMP_DIR}) - -# Where to stage installations during "make" phase. -set(STAGE_DIR ${WORK_DIR}/stage) -set(STAGE_PREFIX_DIR ${STAGE_DIR}${CMAKE_INSTALL_PREFIX}) -set(STAGE_EMBEDDED_DIR ${STAGE_DIR}${INSTALL_PREFIX_EMBEDDED}) -set(HADOOP_ANALYTICS_STAGE_DIR ${WORK_DIR}/stage-hadoop-analytics) -set(HADOOP_ANALYTICS_STAGE_PREFIX_DIR ${HADOOP_ANALYTICS_STAGE_DIR}${CMAKE_INSTALL_PREFIX}) -set(HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR ${HADOOP_ANALYTICS_STAGE_DIR}${INSTALL_PREFIX_EMBEDDED}) - -# Where to install app-level vendor dependencies. -set(VENDOR_DIR ${WORK_DIR}/vendor) -set(VENDOR_LUA_DIR ${VENDOR_DIR}/share/lua/5.1) -if(ENABLE_DEPLOY_ONLY) - set(LUA_PREFIX ${INSTALL_PREFIX_EMBEDDED}) -else() - set(LUA_PREFIX ${STAGE_EMBEDDED_DIR}) -endif() -set(LUAROCKS_CMD env LUA_PATH=${LUA_PREFIX}/openresty/luajit/share/lua/5.1/?.lua$${LUA_PREFIX}/openresty/luajit/share/lua/5.1/?/init.lua$$ ${LUA_PREFIX}/bin/luarocks) - -# Where to install development-only dependencies. -set(DEV_INSTALL_PREFIX ${WORK_DIR}/dev-env) -set(DEV_VENDOR_DIR ${DEV_INSTALL_PREFIX}/vendor) - -# Where to install test-only dependencies. -set(TEST_INSTALL_PREFIX ${WORK_DIR}/test-env) -set(TEST_VENDOR_DIR ${TEST_INSTALL_PREFIX}/vendor) -set(TEST_VENDOR_LUA_SHARE_DIR ${TEST_VENDOR_DIR}/share/lua/5.1) -set(TEST_VENDOR_LUA_LIB_DIR ${TEST_VENDOR_DIR}/lib/lua/5.1) - -# Define a timestamped release name for our app installations. Base this on the -# last git commit timestamp so installs are consistent for each git commit. -include(${CMAKE_SOURCE_DIR}/build/cmake/GetGitRevisionDescription.cmake) -include(${CMAKE_SOURCE_DIR}/build/cmake/GetGitTimestamp.cmake) -get_git_timestamp(RELEASE_TIMESTAMP) -string(SUBSTRING ${RELEASE_TIMESTAMP} 0 8 RELEASE_DATE) - -include(ExternalProject) - -function(require_program name) - find_program(${name} ${name}) - if(NOT ${name}) - MESSAGE(FATAL_ERROR "Could not find ${name}") - endif() -endfunction(require_program) - -if(ENABLE_DEPLOY_ONLY) - # Create stub/empty targets for things the core build process depends on. But - # for deploy-based builds, we'll assume these dependencies have already been - # installed (since we're assuming the deploys are overlaying a package - # installation). - add_custom_target(bundler) - add_custom_target(libcidr) - add_custom_target(luarocks) - - include(${CMAKE_SOURCE_DIR}/build/cmake/versions.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/core.cmake) -else() - find_package(LibXml2 REQUIRED) - find_package(PkgConfig REQUIRED) - pkg_search_module(LIBUUID REQUIRED uuid) - pkg_search_module(LIBFFI REQUIRED libffi) - require_program(rsync) - - include(${CMAKE_SOURCE_DIR}/build/cmake/versions.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/libcidr.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/libgeoip.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/mora.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/openresty.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/luarocks.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/perp.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/runit_svlogd.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/ruby.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/rsyslog.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/trafficserver.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/static-site.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/core.cmake) - if(ENABLE_HADOOP_ANALYTICS) - include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/flume.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/kylin.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/presto.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/hadoop-analytics/processor.cmake) - endif() - - # - # Testing - # - if(ENABLE_TEST_DEPENDENCIES) - include(${CMAKE_SOURCE_DIR}/build/cmake/test/lua-deps.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/test/mailhog.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/test/mongo-orchestration.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/test/openldap.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/test/phantomjs.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/test/shellcheck.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/test/unbound.cmake) - endif() - include(${CMAKE_SOURCE_DIR}/build/cmake/test/bundle.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/test/test.cmake) - - # - # Installation - # - include(${CMAKE_SOURCE_DIR}/build/cmake/install.cmake) - - # - # Packaging - # - include(${CMAKE_SOURCE_DIR}/build/cmake/package.cmake) - - # - # Clean Task - # - include(${CMAKE_SOURCE_DIR}/build/cmake/clean-download-archives.cmake) - include(${CMAKE_SOURCE_DIR}/build/cmake/distclean.cmake) -endif() diff --git a/Dockerfile-beta-deploy b/Dockerfile-beta-deploy new file mode 100644 index 000000000..f42969353 --- /dev/null +++ b/Dockerfile-beta-deploy @@ -0,0 +1,53 @@ +FROM ubuntu:18.04 AS build + +RUN mkdir -p /app/build /build/.task /build/build/work +RUN ln -snf /build/.task /app/.task +RUN ln -snf /build/build/work /app/build/work +WORKDIR /app + +ENV DOCKER_DEV true +ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1 + +COPY tasks/install-system-build-dependencies /app/tasks/install-system-build-dependencies +COPY build/package_dependencies.sh /app/build/package_dependencies.sh +COPY tasks/helpers.sh /app/tasks/helpers.sh +RUN /app/tasks/install-system-build-dependencies + +COPY Makefile.in /app/Makefile.in +COPY Taskfile.yml /app/Taskfile.yml +COPY configure /app/configure +COPY tasks/bootstrap-* /app/tasks/ +RUN ./configure + +COPY build/patches /app/build/patches +COPY tasks/deps /app/tasks/deps +COPY tasks/clean/dev /app/tasks/clean/dev +RUN make deps && make clean:dev + +COPY tasks/build-deps /app/tasks/build-deps +RUN make build-deps && make clean:dev + +COPY src/api-umbrella/admin-ui/.yarnrc /app/src/api-umbrella/admin-ui/.yarnrc +COPY src/api-umbrella/admin-ui/package.json /app/src/api-umbrella/admin-ui/package.json +COPY src/api-umbrella/admin-ui/yarn.lock /app/src/api-umbrella/admin-ui/yarn.lock +COPY src/api-umbrella/web-app/Gemfile /app/src/api-umbrella/web-app/Gemfile +COPY src/api-umbrella/web-app/Gemfile.lock /app/src/api-umbrella/web-app/Gemfile.lock +COPY tasks/app-deps /app/tasks/app-deps +RUN make app-deps && make clean:dev + +COPY . /app +RUN make && make clean:dev +RUN make install DESTDIR="/build/install-destdir" + +FROM ubuntu:18.04 + +COPY --from=build /build/install-destdir / +COPY --from=build /app/build/package_dependencies.sh /tmp/install/package_dependencies.sh +COPY --from=build /app/build/package/scripts/after-install /tmp/install/after-install +RUN set -x && \ + apt-get update && \ + bash -c 'source /tmp/install/package_dependencies.sh && DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install "${core_package_dependencies[@]}"' && \ + /tmp/install/after-install 1 && \ + rm -rf /tmp/install /var/lib/apt/lists/* + +CMD ["api-umbrella", "run"] diff --git a/Dockerfile-build b/Dockerfile-build new file mode 100644 index 000000000..aa3112050 --- /dev/null +++ b/Dockerfile-build @@ -0,0 +1,62 @@ +FROM ubuntu:18.04 AS build + +RUN mkdir -p /app/build /build/.task /build/build/work +RUN ln -snf /build/.task /app/.task +RUN ln -snf /build/build/work /app/build/work +WORKDIR /app + +ENV DOCKER_DEV true +ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1 + +COPY tasks/install-system-build-dependencies /app/tasks/install-system-build-dependencies +COPY build/package_dependencies.sh /app/build/package_dependencies.sh +COPY tasks/helpers.sh /app/tasks/helpers.sh +RUN env INSTALL_TEST_DEPENDENCIES=true /app/tasks/install-system-build-dependencies + +COPY Makefile.in /app/Makefile.in +COPY Taskfile.yml /app/Taskfile.yml +COPY configure /app/configure +COPY tasks/bootstrap-* /app/tasks/ +RUN ./configure + +COPY build/patches /app/build/patches +COPY tasks/deps /app/tasks/deps +COPY tasks/clean/dev /app/tasks/clean/dev +RUN make deps && make clean:dev + +COPY tasks/build-deps /app/tasks/build-deps +RUN make build-deps && make clean:dev + +COPY Gemfile /app/Gemfile +COPY Gemfile.lock /app/Gemfile.lock +COPY tasks/test-deps /app/tasks/test-deps +RUN make test-deps && make clean:dev + +COPY src/api-umbrella/admin-ui/.yarnrc /app/src/api-umbrella/admin-ui/.yarnrc +COPY src/api-umbrella/admin-ui/package.json /app/src/api-umbrella/admin-ui/package.json +COPY src/api-umbrella/admin-ui/yarn.lock /app/src/api-umbrella/admin-ui/yarn.lock +COPY src/api-umbrella/web-app/Gemfile /app/src/api-umbrella/web-app/Gemfile +COPY src/api-umbrella/web-app/Gemfile.lock /app/src/api-umbrella/web-app/Gemfile.lock +COPY tasks/app-deps /app/tasks/app-deps +RUN make app-deps && make clean:dev + +COPY . /app +RUN make && make clean:dev + +RUN rm -rf build/work/tasks + +FROM ubuntu:18.04 + +COPY --from=build /app /app +COPY --from=build /build /build +WORKDIR /app +RUN env INSTALL_TEST_DEPENDENCIES=true /app/tasks/install-system-build-dependencies + +RUN groupadd -r api-umbrella && \ + useradd -r -g api-umbrella -s /sbin/nologin -d /opt/api-umbrella -c "API Umbrella user" api-umbrella + +ENV PATH "/app/bin:/build/build/work/dev-env/sbin:/build/build/work/dev-env/bin:/build/build/work/test-env/sbin:/build/build/work/test-env/bin:/build/build/work/stage/opt/api-umbrella/sbin:/build/build/work/stage/opt/api-umbrella/bin:/build/build/work/stage/opt/api-umbrella/embedded/sbin:/build/build/work/stage/opt/api-umbrella/embedded/bin:${PATH}" +ENV API_UMBRELLA_ROOT /build/build/work/stage/opt/api-umbrella + +ENTRYPOINT ["/app/docker/docker-entrypoint"] +CMD ["/app/docker/dev/docker-start"] diff --git a/Dockerfile-dev b/Dockerfile-dev index b02fb001b..5519905bf 100644 --- a/Dockerfile-dev +++ b/Dockerfile-dev @@ -1,4 +1,4 @@ FROM nrel/api-umbrella-dev-env:latest -RUN cd /build && make WORKDIR /app +RUN ./configure && make diff --git a/Dockerfile-dev-build b/Dockerfile-dev-build index cb8deb657..d393db10f 100644 --- a/Dockerfile-dev-build +++ b/Dockerfile-dev-build @@ -1,34 +1,85 @@ -FROM ubuntu:xenial +FROM ubuntu:18.04 AS build -RUN mkdir /app -COPY build/scripts/install_build_dependencies /app/build/scripts/install_build_dependencies +RUN mkdir -p /app/build /build/.task /build/build/work +RUN ln -snf /build/.task /app/.task +RUN ln -snf /build/build/work /app/build/work +WORKDIR /app + +ENV DOCKER_DEV true +ENV NOKOGIRI_USE_SYSTEM_LIBRARIES 1 + +COPY tasks/install-system-build-dependencies /app/tasks/install-system-build-dependencies COPY build/package_dependencies.sh /app/build/package_dependencies.sh -RUN env INSTALL_TEST_DEPENDENCIES=true /app/build/scripts/install_build_dependencies +COPY tasks/helpers.sh /app/tasks/helpers.sh +RUN env INSTALL_TEST_DEPENDENCIES=true /app/tasks/install-system-build-dependencies + +COPY Makefile.in /app/Makefile.in +COPY Taskfile-dev.yml /app/Taskfile.yml +COPY configure /app/configure +COPY tasks/bootstrap-* /app/tasks/ +RUN ./configure -RUN mkdir /build -WORKDIR /build +COPY build/patches /app/build/patches +COPY tasks/deps /app/tasks/deps +COPY tasks/clean/dev /app/tasks/clean/dev +RUN make deps && make clean:dev -COPY build/scripts/download_cmake /app/build/scripts/download_cmake -RUN /app/build/scripts/download_cmake +COPY tasks/build-deps /app/tasks/build-deps +RUN make build-deps && make clean:dev -COPY CMakeLists.txt /app/CMakeLists.txt COPY Gemfile /app/Gemfile COPY Gemfile.lock /app/Gemfile.lock -COPY build/cmake /app/build/cmake -COPY config/default.yml /app/config/default.yml -COPY configure /app/configure -COPY src/api-umbrella/admin-ui /app/src/api-umbrella/admin-ui -COPY src/api-umbrella/version.txt /app/src/api-umbrella/version.txt -COPY src/api-umbrella/web-app /app/src/api-umbrella/web-app -RUN /app/configure --enable-test-dependencies && make +COPY tasks/test-deps /app/tasks/test-deps +RUN make test-deps && make clean:dev -RUN groupadd -r api-umbrella && \ - useradd -r -g api-umbrella -s /sbin/nologin -d /opt/api-umbrella -c "API Umbrella user" api-umbrella +COPY src/api-umbrella/admin-ui/.yarnrc /app/src/api-umbrella/admin-ui/.yarnrc +COPY src/api-umbrella/admin-ui/package.json /app/src/api-umbrella/admin-ui/package.json +COPY src/api-umbrella/admin-ui/yarn.lock /app/src/api-umbrella/admin-ui/yarn.lock +COPY src/api-umbrella/web-app/Gemfile /app/src/api-umbrella/web-app/Gemfile +COPY src/api-umbrella/web-app/Gemfile.lock /app/src/api-umbrella/web-app/Gemfile.lock +COPY tasks/app-deps /app/tasks/app-deps +RUN make app-deps && make clean:dev COPY . /app +RUN make && make clean:dev -ENV PATH="/app/bin:/build/build/work/dev-env/sbin:/build/build/work/dev-env/bin:/build/build/work/test-env/sbin:/build/build/work/test-env/bin:/build/build/work/stage/opt/api-umbrella/sbin:/build/build/work/stage/opt/api-umbrella/bin:/build/build/work/stage/opt/api-umbrella/embedded/sbin:/build/build/work/stage/opt/api-umbrella/embedded/bin:${PATH}" -ENV API_UMBRELLA_ROOT="/build/build/work/stage/opt/api-umbrella" +RUN rm -rf build/work/tasks +FROM ubuntu:18.04 + +COPY --from=build /app /app +COPY --from=build /build /build WORKDIR /app -CMD ./docker/dev/docker-start +RUN env INSTALL_TEST_DEPENDENCIES=true /app/tasks/install-system-build-dependencies + +# Add Chrome for integration tests, similar to how the CircleCI images add it: +# https://github.com/CircleCI-Public/circleci-dockerfiles/blob/c24e69355b400aaba34a1ddfc55cdb1fef9dedff/buildpack-deps/images/xenial/browsers/Dockerfile#L47 +RUN set -x && \ + apt-get update && \ + curl --silent --show-error --location --fail --retry 3 --output /tmp/google-chrome-stable_current_amd64.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb && \ + (dpkg -i /tmp/google-chrome-stable_current_amd64.deb || apt-get -fy install) && \ + rm -f /tmp/google-chrome-stable_current_amd64.deb && \ + sed -i 's|HERE/chrome"|HERE/chrome" --disable-setuid-sandbox --no-sandbox|g' /opt/google/chrome/google-chrome && \ + google-chrome --version +RUN set -x && \ + CHROME_VERSION="$(google-chrome --version)" && \ + export CHROMEDRIVER_RELEASE="$(echo $CHROME_VERSION | sed 's/^Google Chrome //')" && export CHROMEDRIVER_RELEASE=${CHROMEDRIVER_RELEASE%%.*} && \ + CHROMEDRIVER_VERSION=$(curl --location --fail --retry 3 http://chromedriver.storage.googleapis.com/LATEST_RELEASE_${CHROMEDRIVER_RELEASE}) && \ + curl --silent --show-error --location --fail --retry 3 --output /tmp/chromedriver_linux64.zip "http://chromedriver.storage.googleapis.com/$CHROMEDRIVER_VERSION/chromedriver_linux64.zip" && \ + cd /tmp && \ + unzip chromedriver_linux64.zip && \ + rm -rf chromedriver_linux64.zip && \ + mv chromedriver /usr/local/bin/chromedriver && \ + chmod +x /usr/local/bin/chromedriver && \ + chromedriver --version + +RUN groupadd -r api-umbrella && \ + useradd -r -g api-umbrella -s /sbin/nologin -d /opt/api-umbrella -c "API Umbrella user" api-umbrella + +ENV PATH "/app/bin:/build/build/work/dev-env/sbin:/build/build/work/dev-env/bin:/build/build/work/test-env/sbin:/build/build/work/test-env/bin:/build/build/work/stage/opt/api-umbrella/sbin:/build/build/work/stage/opt/api-umbrella/bin:/build/build/work/stage/opt/api-umbrella/embedded/sbin:/build/build/work/stage/opt/api-umbrella/embedded/bin:${PATH}" +ENV API_UMBRELLA_ROOT /build/build/work/stage/opt/api-umbrella +ENV HTTP_PORT 8080 +ENV HTTPS_PORT 8443 + +ENTRYPOINT ["/app/docker/dev/docker-entrypoint"] +CMD ["/app/docker/dev/docker-start"] diff --git a/Gemfile b/Gemfile index 3e6e841d8..28f90c374 100644 --- a/Gemfile +++ b/Gemfile @@ -9,16 +9,16 @@ gem "minitest", "~> 5.11.3" gem "minitest-sprint", "~> 1.2.0" # More test outputs -gem "minitest-reporters", "~> 1.2.0" +gem "minitest-reporters", "~> 1.3.0" # For an "after_all" callback. -gem "minitest-hooks", "~> 1.4.2" +gem "minitest-hooks", "~> 1.5.0" # Test metadata for CI environment. gem "minitest-ci", "~> 3.4.0" # Ruby lint/style checker -gem "rubocop", "~> 0.54.0", :require => false +gem "rubocop", "~> 0.67.1", :require => false # Running background processes gem "childprocess", "~> 0.9.0" @@ -28,57 +28,55 @@ gem "typhoeus", "~> 1.3.0" # JSON parsing gem "multi_json", "~> 1.13.1" -gem "oj", "~> 3.5.0" +gem "oj", "~> 3.7.0" # Database libraries -gem "mongoid", "~> 6.3.0" -gem "elasticsearch", "~> 2.0.2" -gem "elasticsearch-persistence", "~> 0.1.9" +gem "mongoid", "~> 7.0.1" +gem "elasticsearch", "~> 6.2.0" +gem "active_attr", "~> 0.12.0" # Factories for test database data -gem "factory_bot", "~> 4.8.2" +gem "factory_bot", "~> 5.0.0" # Deleting database data between tests. -gem "database_cleaner", "~> 1.6.2" +gem "database_cleaner", "~> 1.7.0" # Programmatically generate Rails session cookies. gem "rails_compatible_cookies_utils", "~> 0.1.0" # URL parsing/generation -gem "addressable", "~> 2.5.2" +gem "addressable", "~> 2.6.0" # Browser/JavaScript integration tests -gem "capybara", "~> 2.18.0" -# Use fork to fix failure messages: -# https://github.com/wojtekmach/minitest-capybara/pull/17 -gem "minitest-capybara", "~> 0.8.2", :git => "https://github.com/GUI/minitest-capybara.git" - -# Webkit-based driver for capybara -gem "poltergeist", "~> 1.17.0" +gem "capybara", "~> 3.16.1" +gem "selenium-webdriver", "~> 3.141" +gem "webdrivers", "~> 3.7.2", :require => false +# https://github.com/dbalatero/capybara-chromedriver-logger/pull/7 +gem "capybara-chromedriver-logger", "~> 0.3.0", :git => "https://github.com/ThriveTRM/capybara-chromedriver-logger.git", :branch => "do-not-raise-on-filtered-errors" # Take screenshots on capybara test failures -gem "capybara-screenshot", "~> 1.0.14" +gem "capybara-screenshot", "~> 1.0.22" # HTML or XML parsing -gem "nokogiri", "~> 1.8.1" +gem "nokogiri", "~> 1.10.0" # Useful additions -gem "activesupport", "~> 5.1.6" +gem "activesupport", "~> 5.2.0" # Path-based setting of hashes gem "lazyhash", "~> 0.1.1" # Generating fake strings and data. -gem "faker", "~> 1.8.7" +gem "faker", "~> 1.9.1" # Concurrency helpers. -gem "concurrent-ruby", "~> 1.0.5" +gem "concurrent-ruby", "~> 1.1.1" # Time zone randomization for tests. gem "zonebie", "~> 0.6.1" # Encrypting admin passwords. -gem "bcrypt", "~> 3.1.11" +gem "bcrypt", "~> 3.1.12" # Color output gem "rainbow", "~> 3.0.0" diff --git a/Gemfile.lock b/Gemfile.lock index aec750199..3464b3968 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,151 +1,135 @@ GIT - remote: https://github.com/GUI/minitest-capybara.git - revision: 74743d1472891da8636d773557297495cf8d6a77 + remote: https://github.com/ThriveTRM/capybara-chromedriver-logger.git + revision: 77b9c9a11d8ed18719fb2288dd74952bb87b4c44 + branch: do-not-raise-on-filtered-errors specs: - minitest-capybara (0.8.2) - capybara (~> 2.2) - minitest (~> 5.0) - rake + capybara-chromedriver-logger (0.3.0) + capybara + colorize GEM remote: https://rubygems.org/ specs: - activemodel (5.1.6) - activesupport (= 5.1.6) - activesupport (5.1.6) + active_attr (0.12.0) + activemodel (>= 3.0.2, < 6.0) + activesupport (>= 3.0.2, < 6.0) + activemodel (5.2.3) + activesupport (= 5.2.3) + activesupport (5.2.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) - addressable (2.5.2) + addressable (2.6.0) public_suffix (>= 2.0.2, < 4.0) ansi (1.5.0) ast (2.4.0) awesome_print (1.8.0) - axiom-types (0.1.1) - descendants_tracker (~> 0.0.4) - ice_nine (~> 0.11.0) - thread_safe (~> 0.3, >= 0.3.1) - bcrypt (3.1.11) - bson (4.3.0) + bcrypt (3.1.12) + bson (4.4.2) builder (3.2.3) - capybara (2.18.0) + capybara (3.16.1) addressable mini_mime (>= 0.1.3) - nokogiri (>= 1.3.3) - rack (>= 1.0.0) - rack-test (>= 0.5.4) - xpath (>= 2.0, < 4.0) - capybara-screenshot (1.0.18) - capybara (>= 1.0, < 3) + nokogiri (~> 1.8) + rack (>= 1.6.0) + rack-test (>= 0.6.3) + regexp_parser (~> 1.2) + xpath (~> 3.2) + capybara-screenshot (1.0.22) + capybara (>= 1.0, < 4) launchy childprocess (0.9.0) ffi (~> 1.0, >= 1.0.11) - cliver (0.3.2) - coercible (1.0.0) - descendants_tracker (~> 0.0.1) - concurrent-ruby (1.0.5) - database_cleaner (1.6.2) - descendants_tracker (0.0.4) - thread_safe (~> 0.3, >= 0.3.1) - elasticsearch (2.0.2) - elasticsearch-api (= 2.0.2) - elasticsearch-transport (= 2.0.2) - elasticsearch-api (2.0.2) + colorize (0.8.1) + concurrent-ruby (1.1.5) + database_cleaner (1.7.0) + elasticsearch (6.2.0) + elasticsearch-api (= 6.2.0) + elasticsearch-transport (= 6.2.0) + elasticsearch-api (6.2.0) multi_json - elasticsearch-model (5.0.0) - activesupport (> 3) - elasticsearch (> 1) - hashie - elasticsearch-persistence (0.1.9) - activemodel (> 3) - activesupport (> 3) - elasticsearch (> 0.4) - elasticsearch-model (>= 0.1) - hashie - virtus - elasticsearch-transport (2.0.2) + elasticsearch-transport (6.2.0) faraday multi_json - equalizer (0.0.11) - ethon (0.11.0) + ethon (0.12.0) ffi (>= 1.3.0) - factory_bot (4.8.2) - activesupport (>= 3.0.0) - faker (1.8.7) + factory_bot (5.0.2) + activesupport (>= 4.2.0) + faker (1.9.3) i18n (>= 0.7) - faraday (0.14.0) + faraday (0.15.4) multipart-post (>= 1.2, < 3) - ffi (1.9.23) - hashie (3.5.7) - i18n (1.0.0) + ffi (1.10.0) + i18n (1.6.0) concurrent-ruby (~> 1.0) - ice_nine (0.11.2) + jaro_winkler (1.5.2) launchy (2.4.3) addressable (~> 2.3) lazyhash (0.1.1) - mini_mime (1.0.0) - mini_portile2 (2.3.0) + mini_mime (1.0.1) + mini_portile2 (2.4.0) minitest (5.11.3) minitest-ci (3.4.0) minitest (>= 5.0.6) - minitest-hooks (1.4.2) - minitest-reporters (1.2.0) + minitest-hooks (1.5.0) + minitest (> 5.3) + minitest-reporters (1.3.6) ansi builder minitest (>= 5.0) ruby-progressbar minitest-sprint (1.2.0) path_expander (~> 1.0) - mongo (2.5.1) - bson (>= 4.3.0, < 5.0.0) - mongoid (6.3.0) - activemodel (~> 5.1) - mongo (>= 2.5.0, < 3.0.0) + mongo (2.8.0) + bson (>= 4.4.2, < 5.0.0) + mongoid (7.0.2) + activemodel (>= 5.1, < 6.0.0) + mongo (>= 2.5.1, < 3.0.0) multi_json (1.13.1) multipart-post (2.0.0) - nokogiri (1.8.2) - mini_portile2 (~> 2.3.0) - oj (3.5.0) - parallel (1.12.1) - parser (2.5.0.5) + net_http_ssl_fix (0.0.10) + nokogiri (1.10.2) + mini_portile2 (~> 2.4.0) + oj (3.7.11) + parallel (1.17.0) + parser (2.6.2.1) ast (~> 2.4.0) path_expander (1.0.3) - poltergeist (1.17.0) - capybara (~> 2.1) - cliver (~> 0.3.1) - websocket-driver (>= 0.2.0) - powerpack (0.1.1) - public_suffix (3.0.2) - rack (2.0.4) - rack-test (1.0.0) + psych (3.1.0) + public_suffix (3.0.3) + rack (2.0.7) + rack-test (1.1.0) rack (>= 1.0, < 3) rails_compatible_cookies_utils (0.1.2) rainbow (3.0.0) - rake (12.3.1) - rubocop (0.54.0) + rake (12.3.2) + regexp_parser (1.4.0) + rubocop (0.67.2) + jaro_winkler (~> 1.5.1) parallel (~> 1.10) - parser (>= 2.5) - powerpack (~> 0.1) + parser (>= 2.5, != 2.5.1.1) + psych (>= 3.1.0) rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) - unicode-display_width (~> 1.0, >= 1.0.1) - ruby-progressbar (1.9.0) + unicode-display_width (>= 1.4.0, < 1.6) + ruby-progressbar (1.10.0) + rubyzip (1.2.2) + selenium-webdriver (3.141.0) + childprocess (~> 0.5) + rubyzip (~> 1.2, >= 1.2.2) thread_safe (0.3.6) - typhoeus (1.3.0) + typhoeus (1.3.1) ethon (>= 0.9.0) tzinfo (1.2.5) thread_safe (~> 0.1) - unicode-display_width (1.3.0) - virtus (1.0.5) - axiom-types (~> 0.1) - coercible (~> 1.0) - descendants_tracker (~> 0.0, >= 0.0.3) - equalizer (~> 0.0, >= 0.0.9) - websocket-driver (0.7.0) - websocket-extensions (>= 0.1.0) - websocket-extensions (0.1.3) - xpath (3.0.0) + unicode-display_width (1.5.0) + webdrivers (3.7.2) + net_http_ssl_fix + nokogiri (~> 1.6) + rubyzip (~> 1.0) + selenium-webdriver (~> 3.0) + xpath (3.2.0) nokogiri (~> 1.8) zonebie (0.6.1) @@ -153,37 +137,38 @@ PLATFORMS ruby DEPENDENCIES - activesupport (~> 5.1.6) - addressable (~> 2.5.2) + active_attr (~> 0.12.0) + activesupport (~> 5.2.0) + addressable (~> 2.6.0) awesome_print (~> 1.8.0) - bcrypt (~> 3.1.11) - capybara (~> 2.18.0) - capybara-screenshot (~> 1.0.14) + bcrypt (~> 3.1.12) + capybara (~> 3.16.1) + capybara-chromedriver-logger (~> 0.3.0)! + capybara-screenshot (~> 1.0.22) childprocess (~> 0.9.0) - concurrent-ruby (~> 1.0.5) - database_cleaner (~> 1.6.2) - elasticsearch (~> 2.0.2) - elasticsearch-persistence (~> 0.1.9) - factory_bot (~> 4.8.2) - faker (~> 1.8.7) + concurrent-ruby (~> 1.1.1) + database_cleaner (~> 1.7.0) + elasticsearch (~> 6.2.0) + factory_bot (~> 5.0.0) + faker (~> 1.9.1) lazyhash (~> 0.1.1) minitest (~> 5.11.3) - minitest-capybara (~> 0.8.2)! minitest-ci (~> 3.4.0) - minitest-hooks (~> 1.4.2) - minitest-reporters (~> 1.2.0) + minitest-hooks (~> 1.5.0) + minitest-reporters (~> 1.3.0) minitest-sprint (~> 1.2.0) - mongoid (~> 6.3.0) + mongoid (~> 7.0.1) multi_json (~> 1.13.1) - nokogiri (~> 1.8.1) - oj (~> 3.5.0) - poltergeist (~> 1.17.0) + nokogiri (~> 1.10.0) + oj (~> 3.7.0) rails_compatible_cookies_utils (~> 0.1.0) rainbow (~> 3.0.0) rake (~> 12.3.1) - rubocop (~> 0.54.0) + rubocop (~> 0.67.1) + selenium-webdriver (~> 3.141) typhoeus (~> 1.3.0) + webdrivers (~> 3.7.2) zonebie (~> 0.6.1) BUNDLED WITH - 1.16.1 + 1.17.3 diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 000000000..1e09f6bfd --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,36 @@ +pipeline { + agent { + dockerfile { + filename "Dockerfile-dev-build" + } + } + triggers { + cron("H H(0-5) * * *") + } + + environment { + CI = "true" + } + + stages { + stage("build") { + steps { + sh "./configure" + sh "make all test-deps" + sh "make clean:dev" + } + } + + stage("lint") { + steps { + sh "make lint" + } + } + + stage("test") { + steps { + sh "env N=12 make test" + } + } + } +} diff --git a/Makefile.in b/Makefile.in new file mode 100644 index 000000000..f7572e69a --- /dev/null +++ b/Makefile.in @@ -0,0 +1,25 @@ +# A minimal Makefile that downloads the "task" tool and passes things off to +# that tool for executing (so "make all" actually becomes "task all"). See +# Taskfile.yml for more detail. + +ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +WORK_DIR:=$(ROOT_DIR)/build/work + +prefix = @prefix@ +export PREFIX=$(prefix) + +.PHONY: default task-passthrough +default: all + +# Download and locally install the "task" tool. +TASK_VERSION:=2.5.0 +$(ROOT_DIR)/tasks/bootstrap-$(TASK_VERSION): ; +$(WORK_DIR)/stamp/bootstrap-$(TASK_VERSION): $(ROOT_DIR)/tasks/bootstrap-$(TASK_VERSION) + $(ROOT_DIR)/tasks/bootstrap-$(TASK_VERSION) + +task-passthrough: $(WORK_DIR)/stamp/bootstrap-$(TASK_VERSION) + $(WORK_DIR)/task $(MAKECMDGOALS) + +# Match all commands sent to "make" and send them to "task" instead. +%: task-passthrough + @true diff --git a/README.md b/README.md index 2d534fb92..fcafc22a3 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![CircleCI](https://circleci.com/gh/NREL/api-umbrella.svg?style=svg)](https://circleci.com/gh/NREL/api-umbrella) [![Dependency Status](https://gemnasium.com/badges/github.com/NREL/api-umbrella.svg)](https://gemnasium.com/github.com/NREL/api-umbrella) +[![CircleCI](https://circleci.com/gh/NREL/api-umbrella.svg?style=svg)](https://circleci.com/gh/NREL/api-umbrella) # APInf Umbrella diff --git a/Rakefile b/Rakefile index b462c7ef5..15212b2ff 100644 --- a/Rakefile +++ b/Rakefile @@ -15,11 +15,10 @@ ENV["PATH"] = [ "#{API_UMBRELLA_SRC_ROOT}/build/work/test-env/sbin", "#{API_UMBRELLA_SRC_ROOT}/build/work/dev-env/bin", "#{API_UMBRELLA_SRC_ROOT}/build/work/dev-env/sbin", - "#{API_UMBRELLA_SRC_ROOT}/build/work/cmake/bin", ENV["PATH"], ].join(":") Dir.glob(File.join(API_UMBRELLA_SRC_ROOT, "scripts/rake/*.rake")).each { |r| import r } task(:default).clear -task(:default => [:lint, :test]) +task(:default => [:test]) diff --git a/Taskfile-dev.yml b/Taskfile-dev.yml new file mode 100644 index 000000000..58ef3a93d --- /dev/null +++ b/Taskfile-dev.yml @@ -0,0 +1,824 @@ +# Run build tasks, test tasks, and other generic tasks via the go-task tool: +# https://github.com/go-task/task +# +# The primary reason we're using this over something more standard like Make is +# that it provides a simple mechanism to skip tasks based on file checksums. +# This lets us more easily cache and skip steps in the CI and Docker +# development environment (since we only need to cache the build output and +# checksum files, rather than also needing to cache all the intermediate build +# files). +# +# A few general notes on the approach: +# +# - We mostly defer each task to a shell script to do the actual work. +# - For the build process, we use "stamp" files as outputs and also use those +# within the dependency chain. We use these stamp files so that the +# checksumming approach properly invalidates the entire chain of dependencies +# when build files change (whereas just using "deps" in go-task doesn't fully +# work for multiple levels of dependencies). So, for example, if the +# libmaxminddb dependency changes, rebuilding it properly cascades and +# triggers a rebuild of openresty (1st level dependency) and luarocks (2nd +# level dependency). +# - For colorized output we use the "unbuffer" tool to force color output on +# some commands, since Task doesn't easily support detecting colorized output +# (as of the output changes in 2.0.3): +# https://github.com/go-task/task/issues/136 + +version: "2" + +output: interleaved + +tasks: + deps:bundler: + deps: + - deps:rubygems + cmds: + - ./tasks/deps/bundler + sources: + - ./build/work/stamp/deps/rubygems + - ./tasks/deps/bundler + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/bundler + method: checksum + + deps:elasticsearch: + cmds: + - ./tasks/deps/elasticsearch + sources: + - ./tasks/deps/elasticsearch + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/elasticsearch + method: checksum + + deps:geolitecity: + cmds: + - ./tasks/deps/geolitecity + sources: + - ./tasks/deps/geolitecity + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/geolitecity + method: none + + deps:libcidr: + cmds: + - ./tasks/deps/libcidr + sources: + - ./tasks/deps/libcidr + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/libcidr + method: checksum + + deps:libestr: + cmds: + - ./tasks/deps/libestr + sources: + - ./tasks/deps/libestr + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/libestr + method: checksum + + deps:libfastjson: + cmds: + - ./tasks/deps/libfastjson + sources: + - ./tasks/deps/libfastjson + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/libfastjson + method: checksum + + deps:libmaxminddb: + cmds: + - ./tasks/deps/libmaxminddb + sources: + - ./tasks/deps/libmaxminddb + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/libmaxminddb + method: checksum + + deps:luarocks: + deps: + - deps:openresty + cmds: + - ./tasks/deps/luarocks + sources: + - ./build/work/stamp/deps/openresty + - ./tasks/deps/luarocks + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/luarocks + method: checksum + + deps:mongodb: + cmds: + - ./tasks/deps/mongodb + sources: + - ./tasks/deps/mongodb + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/mongodb + method: checksum + + deps:mora: + cmds: + - ./tasks/deps/mora + sources: + - ./tasks/deps/mora + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/mora + method: checksum + + deps:openresty: + deps: + - deps:libmaxminddb + cmds: + - ./tasks/deps/openresty + sources: + - ./build/work/stamp/deps/libmaxminddb + - ./tasks/deps/openresty + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/openresty + method: checksum + + deps:perp: + cmds: + - ./tasks/deps/perp + sources: + - ./tasks/deps/perp + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/perp + method: checksum + + deps:rsyslog: + deps: + - deps:libestr + - deps:libfastjson + cmds: + - ./tasks/deps/rsyslog + sources: + - ./build/work/stamp/deps/libestr + - ./build/work/stamp/deps/libfastjson + - ./tasks/deps/rsyslog + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/rsyslog + method: checksum + + deps:ruby: + cmds: + - ./tasks/deps/ruby + sources: + - ./tasks/deps/ruby + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/ruby + method: checksum + + deps:rubygems: + deps: + - deps:ruby + cmds: + - ./tasks/deps/rubygems + sources: + - ./build/work/stamp/deps/ruby + - ./tasks/deps/rubygems + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/rubygems + method: checksum + + deps:runit_svlogd: + cmds: + - ./tasks/deps/runit_svlogd + sources: + - ./tasks/deps/runit_svlogd + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/runit_svlogd + method: checksum + + deps:trafficserver: + deps: + - deps:openresty + cmds: + - ./tasks/deps/trafficserver + sources: + - ./build/work/stamp/deps/openresty + - ./tasks/deps/trafficserver + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/trafficserver + method: checksum + + deps: + cmds: + - task: deps:bundler + - task: deps:elasticsearch + - task: deps:geolitecity + - task: deps:libcidr + - task: deps:luarocks + - task: deps:mongodb + - task: deps:mora + - task: deps:openresty + - task: deps:perp + - task: deps:rsyslog + - task: deps:runit_svlogd + - task: deps:trafficserver + + build-deps:nodejs: + cmds: + - ./tasks/build-deps/nodejs + sources: + - ./tasks/build-deps/nodejs + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/build-deps/nodejs + method: checksum + + build-deps:yarn: + deps: + - build-deps:nodejs + cmds: + - ./tasks/build-deps/yarn + sources: + - ./build/work/stamp/build-deps/nodejs + - ./tasks/build-deps/yarn + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/build-deps/yarn + method: checksum + + build-deps: + cmds: + - task: build-deps:yarn + + app-deps:admin-ui:yarn: + deps: + - build-deps:yarn + cmds: + - ./tasks/app-deps/admin-ui/yarn + sources: + - ./build/work/stamp/build-deps/yarn + - ./src/api-umbrella/admin-ui/.yarnrc + - ./src/api-umbrella/admin-ui/package.json + - ./src/api-umbrella/admin-ui/yarn.lock + - ./tasks/app-deps/admin-ui/yarn + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/admin-ui/yarn + method: checksum + + app-deps:lua:argparse: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/argparse + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/argparse + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/argparse + method: checksum + + app-deps:lua:cmsgpack: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/cmsgpack + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/cmsgpack + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/cmsgpack + method: checksum + + app-deps:lua:icu-date: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/icu-date + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/icu-date + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/icu-date + method: checksum + + app-deps:lua:inspect: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/inspect + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/inspect + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/inspect + method: checksum + + app-deps:lua:libcidr-ffi: + deps: + - deps:libcidr + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/libcidr-ffi + sources: + - ./build/work/stamp/deps/libcidr + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/libcidr-ffi + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/libcidr-ffi + method: checksum + + + app-deps:lua:luaposix: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/luaposix + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/luaposix + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/luaposix + method: checksum + + app-deps:lua:luasocket: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/luasocket + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/luasocket + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/luasocket + method: checksum + + app-deps:lua:lustache: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/lustache + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/lustache + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/lustache + method: checksum + + app-deps:lua:lyaml: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/lyaml + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/lyaml + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/lyaml + method: checksum + + app-deps:lua:penlight: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/penlight + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/penlight + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/penlight + method: checksum + + app-deps:lua:resty-auto-ssl: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-auto-ssl + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-auto-ssl + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-auto-ssl + method: checksum + + app-deps:lua:resty-http: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-http + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-http + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-http + method: checksum + + app-deps:lua:resty-jwt: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-jwt + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-jwt + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-jwt + method: checksum + + app-deps:lua:resty-logger-socket: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-logger-socket + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-logger-socket + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-logger-socket + method: checksum + + app-deps:lua:resty-shcache: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-shcache + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-shcache + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-shcache + method: checksum + + app-deps:lua:resty-txid: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-txid + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-txid + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-txid + method: checksum + + app-deps:lua:resty-uuid: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-uuid + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-uuid + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-uuid + method: checksum + + app-deps:web-app:bundle: + deps: + - deps:bundler + cmds: + - ./tasks/app-deps/web-app/bundle + sources: + - ./build/work/stamp/deps/bundler + - ./src/api-umbrella/web-app/Gemfile + - ./src/api-umbrella/web-app/Gemfile.lock + - ./tasks/app-deps/web-app/bundle + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/web-app/bundle + method: checksum + + app-deps: + cmds: + - task: app-deps:admin-ui:yarn + - task: app-deps:lua:argparse + - task: app-deps:lua:cmsgpack + - task: app-deps:lua:icu-date + - task: app-deps:lua:inspect + - task: app-deps:lua:libcidr-ffi + - task: app-deps:lua:luaposix + - task: app-deps:lua:luasocket + - task: app-deps:lua:lustache + - task: app-deps:lua:lyaml + - task: app-deps:lua:penlight + - task: app-deps:lua:resty-auto-ssl + - task: app-deps:lua:resty-http + - task: app-deps:lua:resty-jwt + - task: app-deps:lua:resty-logger-socket + - task: app-deps:lua:resty-shcache + - task: app-deps:lua:resty-txid + - task: app-deps:lua:resty-uuid + - task: app-deps:web-app:bundle + + app:admin-ui:build: + deps: + - app-deps:admin-ui:yarn + cmds: + - ./tasks/app/admin-ui/build + sources: + - ./build/work/stamp/app-deps/admin-ui/yarn + - ./src/api-umbrella/admin-ui/app/**/*.hbs + - ./src/api-umbrella/admin-ui/app/**/*.html + - ./src/api-umbrella/admin-ui/app/**/*.js + - ./src/api-umbrella/admin-ui/app/**/*.scss + - ./src/api-umbrella/admin-ui/config/**/*.js + - ./src/api-umbrella/admin-ui/ember-cli-build.js + - ./src/api-umbrella/admin-ui/lib/**/*.js + - ./tasks/app/admin-ui/build + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app/admin-ui/build + method: checksum + + app:core: + deps: + - app:admin-ui:build + - app:web-app:precompile + cmds: + - ./tasks/app/core + sources: + - ./build/work/stamp/app/admin-ui/build + - ./build/work/stamp/app/web-app/precompile + - ./tasks/app/core + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app/core + method: checksum + + app:static-site: + deps: + - build-deps:nodejs + - deps:bundler + cmds: + - ./tasks/app/static-site + sources: + - ./build/work/stamp/build-deps/nodejs + - ./build/work/stamp/deps/bundler + - ./tasks/app/static-site + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app/static-site + method: checksum + + app:web-app:precompile: + deps: + - build-deps:nodejs + - app-deps:web-app:bundle + cmds: + - ./tasks/app/web-app/precompile + sources: + - ./build/work/stamp/app-deps/web-app/bundle + - ./build/work/stamp/build-deps/nodejs + - ./src/api-umbrella/web-app/app/assets/**/*.css + - ./src/api-umbrella/web-app/app/assets/**/*.erb + - ./src/api-umbrella/web-app/app/assets/**/*.js + - ./src/api-umbrella/web-app/app/assets/**/*.scss + - ./tasks/app/web-app/precompile + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app/web-app/precompile + method: checksum + + app: + cmds: + - task: app:core + - task: app:static-site + + all: + cmds: + - task: build-deps + - task: deps + - task: app-deps + - task: app + + test-deps:bundle: + deps: + - deps:bundler + cmds: + - ./tasks/test-deps/bundle + sources: + - ./build/work/stamp/deps/bundler + - ./Gemfile + - ./Gemfile.lock + - ./tasks/test-deps/bundle + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/bundle + method: checksum + + test-deps:elasticsearch5: + cmds: + - ./tasks/test-deps/elasticsearch5 + sources: + - ./tasks/test-deps/elasticsearch5 + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/elasticsearch5 + method: checksum + + test-deps:elasticsearch6: + cmds: + - ./tasks/test-deps/elasticsearch6 + sources: + - ./tasks/test-deps/elasticsearch6 + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/elasticsearch6 + method: checksum + + test-deps:lua:luacheck: + deps: + - deps:luarocks + cmds: + - ./tasks/test-deps/lua/luacheck + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/test-deps/lua/luacheck + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/lua/luacheck + method: checksum + + test-deps:lua:penlight: + deps: + - deps:luarocks + cmds: + - ./tasks/test-deps/lua/penlight + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/test-deps/lua/penlight + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/lua/penlight + method: checksum + + test-deps:mailhog: + cmds: + - ./tasks/test-deps/mailhog + sources: + - ./tasks/test-deps/mailhog + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/mailhog + method: checksum + + test-deps:mongo-orchestration: + cmds: + - ./tasks/test-deps/mongo-orchestration + sources: + - ./tasks/test-deps/mongo-orchestration + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/mongo-orchestration + method: checksum + + test-deps:openldap: + cmds: + - ./tasks/test-deps/openldap + sources: + - ./tasks/test-deps/openldap + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/openldap + method: checksum + + test-deps:shellcheck: + cmds: + - ./tasks/test-deps/shellcheck + sources: + - ./tasks/test-deps/shellcheck + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/shellcheck + method: checksum + + test-deps:unbound: + cmds: + - ./tasks/test-deps/unbound + sources: + - ./tasks/test-deps/unbound + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/unbound + method: checksum + + test-deps: + cmds: + - task: test-deps:bundle + - task: test-deps:elasticsearch5 + - task: test-deps:elasticsearch6 + - task: test-deps:lua:luacheck + - task: test-deps:lua:penlight + - task: test-deps:mailhog + - task: test-deps:mongo-orchestration + - task: test-deps:openldap + - task: test-deps:shellcheck + - task: test-deps:unbound + + lint:js: + deps: + - app-deps:admin-ui:yarn + cmds: + - unbuffer ./tasks/lint/js + + lint:lua: + deps: + - test-deps:lua:luacheck + cmds: + - unbuffer ./tasks/lint/lua + + lint:resty: + deps: + - test-deps:lua:penlight + cmds: + - unbuffer ./tasks/lint/resty/run + + lint:ruby: + deps: + - test-deps:bundle + cmds: + - unbuffer ./tasks/lint/ruby + + lint:shell: + deps: + - test-deps:shellcheck + cmds: + - unbuffer ./tasks/lint/shell + + lint: + cmds: + - task: lint:js + - task: lint:lua + - task: lint:resty + - task: lint:ruby + - task: lint:shell + + outdated: + cmds: + - unbuffer ./tasks/outdated + + test: + deps: + - all + - test-deps + cmds: + - unbuffer ./tasks/test/default + + test:circle-ci: + deps: + - all + - test-deps + cmds: + - ./tasks/test/circle-ci + + clean:dev: + cmds: + - ./tasks/clean/dev + + distclean: + cmds: + - ./tasks/distclean + + install: + cmds: + - ./tasks/install + + package: + cmds: + - ./tasks/package + + install-system-build-dependencies: + cmds: + - ./tasks/install-system-build-dependencies + + default: + cmds: + - task: all diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 000000000..d27ae01cb --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,766 @@ +# Run build tasks, test tasks, and other generic tasks via the go-task tool: +# https://github.com/go-task/task +# +# The primary reason we're using this over something more standard like Make is +# that it provides a simple mechanism to skip tasks based on file checksums. +# This lets us more easily cache and skip steps in the CI and Docker +# development environment (since we only need to cache the build output and +# checksum files, rather than also needing to cache all the intermediate build +# files). +# +# A few general notes on the approach: +# +# - We mostly defer each task to a shell script to do the actual work. +# - For the build process, we use "stamp" files as outputs and also use those +# within the dependency chain. We use these stamp files so that the +# checksumming approach properly invalidates the entire chain of dependencies +# when build files change (whereas just using "deps" in go-task doesn't fully +# work for multiple levels of dependencies). So, for example, if the +# libmaxminddb dependency changes, rebuilding it properly cascades and +# triggers a rebuild of openresty (1st level dependency) and luarocks (2nd +# level dependency). +# - For colorized output we use the "unbuffer" tool to force color output on +# some commands, since Task doesn't easily support detecting colorized output +# (as of the output changes in 2.0.3): +# https://github.com/go-task/task/issues/136 + +version: "2" + +output: interleaved + +tasks: + deps:bundler: + deps: + - deps:rubygems + cmds: + - ./tasks/deps/bundler + sources: + - ./build/work/stamp/deps/rubygems + - ./tasks/deps/bundler + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/bundler + method: checksum + + deps:geolitecity: + cmds: + - ./tasks/deps/geolitecity + sources: + - ./tasks/deps/geolitecity + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/geolitecity + method: none + + deps:libcidr: + cmds: + - ./tasks/deps/libcidr + sources: + - ./tasks/deps/libcidr + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/libcidr + method: checksum + + deps:libestr: + cmds: + - ./tasks/deps/libestr + sources: + - ./tasks/deps/libestr + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/libestr + method: checksum + + deps:libfastjson: + cmds: + - ./tasks/deps/libfastjson + sources: + - ./tasks/deps/libfastjson + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/libfastjson + method: checksum + + deps:libmaxminddb: + cmds: + - ./tasks/deps/libmaxminddb + sources: + - ./tasks/deps/libmaxminddb + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/libmaxminddb + method: checksum + + deps:luarocks: + deps: + - deps:openresty + cmds: + - ./tasks/deps/luarocks + sources: + - ./build/work/stamp/deps/openresty + - ./tasks/deps/luarocks + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/luarocks + method: checksum + + deps:mora: + cmds: + - ./tasks/deps/mora + sources: + - ./tasks/deps/mora + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/mora + method: checksum + + deps:openresty: + deps: + - deps:libmaxminddb + cmds: + - ./tasks/deps/openresty + sources: + - ./build/work/stamp/deps/libmaxminddb + - ./tasks/deps/openresty + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/openresty + method: checksum + + deps:perp: + cmds: + - ./tasks/deps/perp + sources: + - ./tasks/deps/perp + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/perp + method: checksum + + deps:rsyslog: + deps: + - deps:libestr + - deps:libfastjson + cmds: + - ./tasks/deps/rsyslog + sources: + - ./build/work/stamp/deps/libestr + - ./build/work/stamp/deps/libfastjson + - ./tasks/deps/rsyslog + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/rsyslog + method: checksum + + deps:ruby: + cmds: + - ./tasks/deps/ruby + sources: + - ./tasks/deps/ruby + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/ruby + method: checksum + + deps:rubygems: + deps: + - deps:ruby + cmds: + - ./tasks/deps/rubygems + sources: + - ./build/work/stamp/deps/ruby + - ./tasks/deps/rubygems + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/rubygems + method: checksum + + deps:runit_svlogd: + cmds: + - ./tasks/deps/runit_svlogd + sources: + - ./tasks/deps/runit_svlogd + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/runit_svlogd + method: checksum + + deps:trafficserver: + deps: + - deps:openresty + cmds: + - ./tasks/deps/trafficserver + sources: + - ./build/work/stamp/deps/openresty + - ./tasks/deps/trafficserver + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/deps/trafficserver + method: checksum + + deps: + cmds: + - task: deps:bundler + - task: deps:geolitecity + - task: deps:libcidr + - task: deps:luarocks + - task: deps:mora + - task: deps:openresty + - task: deps:perp + - task: deps:rsyslog + - task: deps:runit_svlogd + - task: deps:trafficserver + + build-deps:nodejs: + cmds: + - ./tasks/build-deps/nodejs + sources: + - ./tasks/build-deps/nodejs + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/build-deps/nodejs + method: checksum + + build-deps:yarn: + deps: + - build-deps:nodejs + cmds: + - ./tasks/build-deps/yarn + sources: + - ./build/work/stamp/build-deps/nodejs + - ./tasks/build-deps/yarn + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/build-deps/yarn + method: checksum + + build-deps: + cmds: + - task: build-deps:yarn + + app-deps:admin-ui:yarn: + deps: + - build-deps:yarn + cmds: + - ./tasks/app-deps/admin-ui/yarn + sources: + - ./build/work/stamp/build-deps/yarn + - ./src/api-umbrella/admin-ui/.yarnrc + - ./src/api-umbrella/admin-ui/package.json + - ./src/api-umbrella/admin-ui/yarn.lock + - ./tasks/app-deps/admin-ui/yarn + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/admin-ui/yarn + method: checksum + + app-deps:lua:argparse: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/argparse + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/argparse + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/argparse + method: checksum + + app-deps:lua:cmsgpack: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/cmsgpack + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/cmsgpack + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/cmsgpack + method: checksum + + app-deps:lua:icu-date: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/icu-date + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/icu-date + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/icu-date + method: checksum + + app-deps:lua:inspect: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/inspect + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/inspect + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/inspect + method: checksum + + app-deps:lua:libcidr-ffi: + deps: + - deps:libcidr + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/libcidr-ffi + sources: + - ./build/work/stamp/deps/libcidr + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/libcidr-ffi + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/libcidr-ffi + method: checksum + + + app-deps:lua:luaposix: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/luaposix + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/luaposix + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/luaposix + method: checksum + + app-deps:lua:luasocket: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/luasocket + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/luasocket + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/luasocket + method: checksum + + app-deps:lua:lustache: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/lustache + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/lustache + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/lustache + method: checksum + + app-deps:lua:lyaml: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/lyaml + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/lyaml + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/lyaml + method: checksum + + app-deps:lua:penlight: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/penlight + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/penlight + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/penlight + method: checksum + + app-deps:lua:resty-auto-ssl: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-auto-ssl + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-auto-ssl + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-auto-ssl + method: checksum + + app-deps:lua:resty-http: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-http + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-http + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-http + method: checksum + + app-deps:lua:resty-jwt: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-jwt + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-jwt + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-jwt + method: checksum + + app-deps:lua:resty-logger-socket: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-logger-socket + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-logger-socket + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-logger-socket + method: checksum + + app-deps:lua:resty-shcache: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-shcache + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-shcache + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-shcache + method: checksum + + app-deps:lua:resty-txid: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-txid + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-txid + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-txid + method: checksum + + app-deps:lua:resty-uuid: + deps: + - deps:luarocks + cmds: + - ./tasks/app-deps/lua/resty-uuid + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/app-deps/lua/resty-uuid + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/lua/resty-uuid + method: checksum + + app-deps:web-app:bundle: + deps: + - deps:bundler + cmds: + - ./tasks/app-deps/web-app/bundle + sources: + - ./build/work/stamp/deps/bundler + - ./src/api-umbrella/web-app/Gemfile + - ./src/api-umbrella/web-app/Gemfile.lock + - ./tasks/app-deps/web-app/bundle + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app-deps/web-app/bundle + method: checksum + + app-deps: + cmds: + - task: app-deps:admin-ui:yarn + - task: app-deps:lua:argparse + - task: app-deps:lua:cmsgpack + - task: app-deps:lua:icu-date + - task: app-deps:lua:inspect + - task: app-deps:lua:libcidr-ffi + - task: app-deps:lua:luaposix + - task: app-deps:lua:luasocket + - task: app-deps:lua:lustache + - task: app-deps:lua:lyaml + - task: app-deps:lua:penlight + - task: app-deps:lua:resty-auto-ssl + - task: app-deps:lua:resty-http + - task: app-deps:lua:resty-jwt + - task: app-deps:lua:resty-logger-socket + - task: app-deps:lua:resty-shcache + - task: app-deps:lua:resty-txid + - task: app-deps:lua:resty-uuid + - task: app-deps:web-app:bundle + + app:admin-ui:build: + deps: + - app-deps:admin-ui:yarn + cmds: + - ./tasks/app/admin-ui/build + sources: + - ./build/work/stamp/app-deps/admin-ui/yarn + - ./src/api-umbrella/admin-ui/app/**/*.hbs + - ./src/api-umbrella/admin-ui/app/**/*.html + - ./src/api-umbrella/admin-ui/app/**/*.js + - ./src/api-umbrella/admin-ui/app/**/*.scss + - ./src/api-umbrella/admin-ui/config/**/*.js + - ./src/api-umbrella/admin-ui/ember-cli-build.js + - ./src/api-umbrella/admin-ui/lib/**/*.js + - ./tasks/app/admin-ui/build + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app/admin-ui/build + method: checksum + + app:core: + deps: + - app:admin-ui:build + - app:web-app:precompile + cmds: + - ./tasks/app/core + sources: + - ./build/work/stamp/app/admin-ui/build + - ./build/work/stamp/app/web-app/precompile + - ./tasks/app/core + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app/core + method: checksum + + app:static-site: + deps: + - build-deps:nodejs + - deps:bundler + cmds: + - ./tasks/app/static-site + sources: + - ./build/work/stamp/build-deps/nodejs + - ./build/work/stamp/deps/bundler + - ./tasks/app/static-site + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app/static-site + method: checksum + + app:web-app:precompile: + deps: + - build-deps:nodejs + - app-deps:web-app:bundle + cmds: + - ./tasks/app/web-app/precompile + sources: + - ./build/work/stamp/app-deps/web-app/bundle + - ./build/work/stamp/build-deps/nodejs + - ./src/api-umbrella/web-app/app/assets/**/*.css + - ./src/api-umbrella/web-app/app/assets/**/*.erb + - ./src/api-umbrella/web-app/app/assets/**/*.js + - ./src/api-umbrella/web-app/app/assets/**/*.scss + - ./tasks/app/web-app/precompile + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/app/web-app/precompile + method: checksum + + app: + cmds: + - task: app:core + - task: app:static-site + + all: + cmds: + - task: build-deps + - task: deps + - task: app-deps + - task: app + + test-deps:bundle: + deps: + - deps:bundler + cmds: + - ./tasks/test-deps/bundle + sources: + - ./build/work/stamp/deps/bundler + - ./Gemfile + - ./Gemfile.lock + - ./tasks/test-deps/bundle + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/bundle + method: checksum + + test-deps:lua:luacheck: + deps: + - deps:luarocks + cmds: + - ./tasks/test-deps/lua/luacheck + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/test-deps/lua/luacheck + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/lua/luacheck + method: checksum + + test-deps:lua:penlight: + deps: + - deps:luarocks + cmds: + - ./tasks/test-deps/lua/penlight + sources: + - ./build/work/stamp/deps/luarocks + - ./tasks/test-deps/lua/penlight + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/lua/penlight + method: checksum + + test-deps:mailhog: + cmds: + - ./tasks/test-deps/mailhog + sources: + - ./tasks/test-deps/mailhog + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/mailhog + method: checksum + + test-deps:mongo-orchestration: + cmds: + - ./tasks/test-deps/mongo-orchestration + sources: + - ./tasks/test-deps/mongo-orchestration + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/mongo-orchestration + method: checksum + + test-deps:openldap: + cmds: + - ./tasks/test-deps/openldap + sources: + - ./tasks/test-deps/openldap + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/openldap + method: checksum + + test-deps:shellcheck: + cmds: + - ./tasks/test-deps/shellcheck + sources: + - ./tasks/test-deps/shellcheck + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/shellcheck + method: checksum + + test-deps:unbound: + cmds: + - ./tasks/test-deps/unbound + sources: + - ./tasks/test-deps/unbound + - ./tasks/helpers.sh + generates: + - ./build/work/stamp/test-deps/unbound + method: checksum + + test-deps: + cmds: + - task: test-deps:bundle + - task: test-deps:lua:luacheck + - task: test-deps:lua:penlight + - task: test-deps:mailhog + - task: test-deps:mongo-orchestration + - task: test-deps:openldap + - task: test-deps:shellcheck + - task: test-deps:unbound + + lint:js: + deps: + - app-deps:admin-ui:yarn + cmds: + - unbuffer ./tasks/lint/js + + lint:lua: + deps: + - test-deps:lua:luacheck + cmds: + - unbuffer ./tasks/lint/lua + + lint:resty: + deps: + - test-deps:lua:penlight + cmds: + - unbuffer ./tasks/lint/resty/run + + lint:ruby: + deps: + - test-deps:bundle + cmds: + - unbuffer ./tasks/lint/ruby + + lint:shell: + deps: + - test-deps:shellcheck + cmds: + - unbuffer ./tasks/lint/shell + + lint: + cmds: + - task: lint:js + - task: lint:lua + - task: lint:resty + - task: lint:ruby + - task: lint:shell + + outdated: + cmds: + - unbuffer ./tasks/outdated + + clean:dev: + cmds: + - ./tasks/clean/dev + + distclean: + cmds: + - ./tasks/distclean + + install: + cmds: + - ./tasks/install + + package: + cmds: + - ./tasks/package + + install-system-build-dependencies: + cmds: + - ./tasks/install-system-build-dependencies + + default: + cmds: + - task: all diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 507117289..000000000 --- a/Vagrantfile +++ /dev/null @@ -1,105 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -options = { - # Allow NFS file sharing to be disabled - :nfs => (ENV["API_UMBRELLA_VAGRANT_NFS"] == "true") || !Vagrant::Util::Platform.windows?, - - # Allow picking a different Vagrant base box: - # API_UMBRELLA_VAGRANT_BOX="chef/debian-7.4" vagrant up - :box => ENV["API_UMBRELLA_VAGRANT_BOX"] || "nrel/CentOS-6.7-x86_64", - - # Allow adjusting the memory and cores when starting the VM: - :memory => (ENV["API_UMBRELLA_VAGRANT_MEMORY"] || "2048").to_i, - :cores => (ENV["API_UMBRELLA_VAGRANT_CORES"] || "2").to_i, - - # Allow a different IP - :ip => ENV["API_UMBRELLA_VAGRANT_IP"] || "10.10.33.2", -} - -plugins = { "vagrant-berkshelf" => nil } - -plugins.each do |plugin, version| - unless(Vagrant.has_plugin?(plugin)) - error = "The '#{plugin}' plugin is not installed. Try running:\n" - error << "vagrant plugin install #{plugin}" - error << " --plugin-version #{version}" if(version) - raise error - end -end - -Vagrant.configure("2") do |config| - # All Vagrant configuration is done here. The most common configuration - # options are documented and commented below. For a complete reference, - # please see the online documentation at vagrantup.com. - - # Every Vagrant virtual environment requires a box to build off of. - config.vm.box = options[:box] - - # Boot with a GUI so you can see the screen. (Default is headless) - # config.vm.boot_mode = :gui - - # Assign a hostname unique to this project. - config.vm.hostname = "api.vagrant" - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - config.vm.network :private_network, :ip => options[:ip] - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network :public_network - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - config.vm.synced_folder ".", "/vagrant", :nfs => options[:nfs] - if(options[:nfs]) - config.nfs.map_uid = Process.uid - config.nfs.map_gid = Process.gid - end - - config.vm.synced_folder "src/api-umbrella/admin-ui", "/vagrant-admin-ui", - :type => "rsync", - :rsync__verbose => true, - :rsync__exclude => [ - "tmp", - "node_modules", - "dist", - ] - - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. - config.vm.provider :virtualbox do |vb| - # Adjust memory used by the VM. - vb.customize ["modifyvm", :id, "--memory", options[:memory]] - vb.customize ["modifyvm", :id, "--cpus", options[:cores]] - - # Keep the virtual machine's clock better in sync to prevent drift (by - # default VirtualBox only syncs if the clocks get more than 20 minutes out - # of sync). - vb.customize ["guestproperty", "set", :id, "/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold", 1000] - end - - # Use the user's local SSH keys for git access. - config.ssh.forward_agent = true - - # Provision the development environment with our Chef cookbook. - config.vm.provision :chef_solo do |chef| - chef.run_list = [ - "recipe[api-umbrella::development]", - ] - end - - # Always restart API Umbrella after starting the machine. This ensures the - # development version get started from the /vagrant partition (since the - # /vagrant NFS partition isn't started early enough during normal boot, we - # must do this here). - config.vm.provision :shell, :run => "always", :inline => <<-eos - if [ -f /etc/init.d/api-umbrella ]; then - /etc/init.d/api-umbrella restart - fi - eos -end diff --git a/bin/api-umbrella-env b/bin/api-umbrella-env index facfe4802..491596127 100644 --- a/bin/api-umbrella-env +++ b/bin/api-umbrella-env @@ -59,8 +59,9 @@ export LD_LIBRARY_PATH="$API_UMBRELLA_EMBEDDED_ROOT/openresty/luajit/lib:$API_UM # Note that we purposefully don't use any of the default Lua load paths (like # /usr/local) so that other Lua packages on the system aren't picked up (since # they might conflict). -export LUA_PATH="$API_UMBRELLA_EMBEDDED_ROOT/apps/core/shared/vendor/share/lua/5.1/?.lua;\ -$API_UMBRELLA_EMBEDDED_ROOT/apps/core/shared/vendor/share/lua/5.1/?/init.lua;\ +export LUA_PATH="$API_UMBRELLA_EMBEDDED_ROOT/apps/core/shared/vendor/lua/resty_modules/lualib/?.lua;\ +$API_UMBRELLA_EMBEDDED_ROOT/apps/core/shared/vendor/lua/share/lua/5.1/?.lua;\ +$API_UMBRELLA_EMBEDDED_ROOT/apps/core/shared/vendor/lua/share/lua/5.1/?/init.lua;\ $API_UMBRELLA_EMBEDDED_ROOT/openresty/lualib/?.lua;\ $API_UMBRELLA_EMBEDDED_ROOT/openresty/lualib/?/init.lua;\ $API_UMBRELLA_EMBEDDED_ROOT/openresty/luajit/share/luajit-2.1.0-beta2/?.lua;\ @@ -70,7 +71,8 @@ $API_UMBRELLA_EMBEDDED_ROOT/openresty/luajit/share/lua/5.1/?/init.lua" if [ -n "$API_UMBRELLA_SRC_ROOT" ]; then export LUA_PATH="$API_UMBRELLA_SRC_ROOT/src/?.lua;$LUA_PATH" fi -export LUA_CPATH="$API_UMBRELLA_EMBEDDED_ROOT/apps/core/shared/vendor/lib/lua/5.1/?.so;\ +export LUA_CPATH="$API_UMBRELLA_EMBEDDED_ROOT/apps/core/shared/vendor/lua/resty_modules/?.so;\ +$API_UMBRELLA_EMBEDDED_ROOT/apps/core/shared/vendor/lua/lib/lua/5.1/?.so;\ $API_UMBRELLA_EMBEDDED_ROOT/openresty/lualib/?.so;\ $API_UMBRELLA_EMBEDDED_ROOT/openresty/luajit/lib/lua/5.1/?.so" diff --git a/bin/api-umbrella-geoip-auto-updater b/bin/api-umbrella-geoip-auto-updater index 10d347a3e..80b3c3cfa 100755 --- a/bin/api-umbrella-geoip-auto-updater +++ b/bin/api-umbrella-geoip-auto-updater @@ -22,11 +22,11 @@ if [ -z "${API_UMBRELLA_DB_DIR:-}" ]; then exit 1 fi -download_url="https://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz" +download_url="https://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz" frequency=86400 # Sleep for 1 day between runs while true; do - current_path="$API_UMBRELLA_DB_DIR/geoip/city-v6.dat" + current_path="$API_UMBRELLA_DB_DIR/geoip/GeoLite2-City.mmdb" # Don't attempt to redownload if the current file has recently been updated # (within the last 22 hours). @@ -38,15 +38,16 @@ while true; do # Download the data file to a temporary path. echo "Downloading new file..." - unzip_path=$(mktemp -t api-umbrella-geoip-auto-updater.XXXXXXXXXX) - download_path="$unzip_path.gz" - if ! curl --silent --show-error --fail --location --output "$download_path" "$download_url"; then + unzip_dir=$(mktemp -d -t api-umbrella-geoip-auto-updater.XXXXXXXXXX) + download_path="$unzip_dir.gz" + if ! curl --silent --show-error --fail --location --retry 3 --output "$download_path" "$download_url"; then echo "Error downloading $download_url" else # Un-gzip the downloaded file. - if ! gunzip -c "$download_path" > "$unzip_path"; then + if ! tar -xf "$download_path" -C "$unzip_dir" --strip-components 1; then echo "Error unzipping $download_path" else + unzip_path="$unzip_dir/GeoLite2-City.mmdb" unzip_md5=$(openssl md5 "$unzip_path" | awk '{print $2}') if [ -f "$current_path" ]; then current_md5=$(openssl md5 "$current_path" | awk '{print $2}') diff --git a/bin/api-umbrella-nginx-reloader b/bin/api-umbrella-nginx-reloader deleted file mode 100755 index 993875555..000000000 --- a/bin/api-umbrella-nginx-reloader +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -e -u - -# Kill the child "sleep" processes on exit. -cleanup() { - local pids - mapfile -t pids < <(jobs -pr) - if [ -n "${pids:-}" ]; then - kill "${pids[@]}" - fi -} -trap "cleanup" EXIT - -if [ -z "${API_UMBRELLA_NGINX_RELOADER_FREQUENCY:-}" ]; then - echo "Error: API_UMBRELLA_NGINX_RELOADER_FREQUENCY environment variable is not set" - exit 1 -fi - -if [ -z "${API_UMBRELLA_PERP_BASE:-}" ]; then - echo "Error: API_UMBRELLA_PERP_BASE environment variable is not set" - exit 1 -fi - -# FIXME: Reload nginx after a certain amount of time passes. This is an ugly -# workaround to alleviate potential memory leaks we've seen in some -# environments: https://github.com/18F/api.data.gov/issues/296 We should remove -# this once we figure out the underlying issue. -while true; do - sleep "$API_UMBRELLA_NGINX_RELOADER_FREQUENCY" - echo "Reloading nginx..." - perpctl -b "$API_UMBRELLA_PERP_BASE" hup nginx -done diff --git a/build/cmake/GetGitRevisionDescription.cmake b/build/cmake/GetGitRevisionDescription.cmake deleted file mode 100644 index 85eae1562..000000000 --- a/build/cmake/GetGitRevisionDescription.cmake +++ /dev/null @@ -1,130 +0,0 @@ -# - Returns a version string from Git -# -# These functions force a re-configure on each git commit so that you can -# trust the values of the variables in your build system. -# -# get_git_head_revision( [ ...]) -# -# Returns the refspec and sha hash of the current head revision -# -# git_describe( [ ...]) -# -# Returns the results of git describe on the source tree, and adjusting -# the output so that it tests false if an error occurs. -# -# git_get_exact_tag( [ ...]) -# -# Returns the results of git describe --exact-match on the source tree, -# and adjusting the output so that it tests false if there was no exact -# matching tag. -# -# Requires CMake 2.6 or newer (uses the 'function' command) -# -# Original Author: -# 2009-2010 Ryan Pavlik -# http://academic.cleardefinition.com -# Iowa State University HCI Graduate Program/VRAC -# -# Copyright Iowa State University 2009-2010. -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -if(__get_git_revision_description) - return() -endif() -set(__get_git_revision_description YES) - -# We must run the following at "include" time, not at function call time, -# to find the path to this module rather than the path to a calling list file -get_filename_component(_gitdescmoddir ${CMAKE_CURRENT_LIST_FILE} PATH) - -function(get_git_head_revision _refspecvar _hashvar) - set(GIT_PARENT_DIR "${CMAKE_CURRENT_SOURCE_DIR}") - set(GIT_DIR "${GIT_PARENT_DIR}/.git") - while(NOT EXISTS "${GIT_DIR}") # .git dir not found, search parent directories - set(GIT_PREVIOUS_PARENT "${GIT_PARENT_DIR}") - get_filename_component(GIT_PARENT_DIR ${GIT_PARENT_DIR} PATH) - if(GIT_PARENT_DIR STREQUAL GIT_PREVIOUS_PARENT) - # We have reached the root directory, we are not in git - set(${_refspecvar} "GITDIR-NOTFOUND" PARENT_SCOPE) - set(${_hashvar} "GITDIR-NOTFOUND" PARENT_SCOPE) - return() - endif() - set(GIT_DIR "${GIT_PARENT_DIR}/.git") - endwhile() - # check if this is a submodule - if(NOT IS_DIRECTORY ${GIT_DIR}) - file(READ ${GIT_DIR} submodule) - string(REGEX REPLACE "gitdir: (.*)\n$" "\\1" GIT_DIR_RELATIVE ${submodule}) - get_filename_component(SUBMODULE_DIR ${GIT_DIR} PATH) - get_filename_component(GIT_DIR ${SUBMODULE_DIR}/${GIT_DIR_RELATIVE} ABSOLUTE) - endif() - set(GIT_DATA "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/git-data") - if(NOT EXISTS "${GIT_DATA}") - file(MAKE_DIRECTORY "${GIT_DATA}") - endif() - - if(NOT EXISTS "${GIT_DIR}/HEAD") - return() - endif() - set(HEAD_FILE "${GIT_DATA}/HEAD") - configure_file("${GIT_DIR}/HEAD" "${HEAD_FILE}" COPYONLY) - - configure_file("${_gitdescmoddir}/GetGitRevisionDescription.cmake.in" - "${GIT_DATA}/grabRef.cmake" - @ONLY) - include("${GIT_DATA}/grabRef.cmake") - - set(${_refspecvar} "${HEAD_REF}" PARENT_SCOPE) - set(${_hashvar} "${HEAD_HASH}" PARENT_SCOPE) -endfunction() - -function(git_describe _var) - if(NOT GIT_FOUND) - find_package(Git QUIET) - endif() - get_git_head_revision(refspec hash) - if(NOT GIT_FOUND) - set(${_var} "GIT-NOTFOUND" PARENT_SCOPE) - return() - endif() - if(NOT hash) - set(${_var} "HEAD-HASH-NOTFOUND" PARENT_SCOPE) - return() - endif() - - # TODO sanitize - #if((${ARGN}" MATCHES "&&") OR - # (ARGN MATCHES "||") OR - # (ARGN MATCHES "\\;")) - # message("Please report the following error to the project!") - # message(FATAL_ERROR "Looks like someone's doing something nefarious with git_describe! Passed arguments ${ARGN}") - #endif() - - #message(STATUS "Arguments to execute_process: ${ARGN}") - - execute_process(COMMAND - "${GIT_EXECUTABLE}" - describe - ${hash} - ${ARGN} - WORKING_DIRECTORY - "${CMAKE_CURRENT_SOURCE_DIR}" - RESULT_VARIABLE - res - OUTPUT_VARIABLE - out - ERROR_QUIET - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT res EQUAL 0) - set(out "${out}-${res}-NOTFOUND") - endif() - - set(${_var} "${out}" PARENT_SCOPE) -endfunction() - -function(git_get_exact_tag _var) - git_describe(out --exact-match ${ARGN}) - set(${_var} "${out}" PARENT_SCOPE) -endfunction() diff --git a/build/cmake/GetGitRevisionDescription.cmake.in b/build/cmake/GetGitRevisionDescription.cmake.in deleted file mode 100644 index 6d8b708ef..000000000 --- a/build/cmake/GetGitRevisionDescription.cmake.in +++ /dev/null @@ -1,41 +0,0 @@ -# -# Internal file for GetGitRevisionDescription.cmake -# -# Requires CMake 2.6 or newer (uses the 'function' command) -# -# Original Author: -# 2009-2010 Ryan Pavlik -# http://academic.cleardefinition.com -# Iowa State University HCI Graduate Program/VRAC -# -# Copyright Iowa State University 2009-2010. -# Distributed under the Boost Software License, Version 1.0. -# (See accompanying file LICENSE_1_0.txt or copy at -# http://www.boost.org/LICENSE_1_0.txt) - -set(HEAD_HASH) - -file(READ "@HEAD_FILE@" HEAD_CONTENTS LIMIT 1024) - -string(STRIP "${HEAD_CONTENTS}" HEAD_CONTENTS) -if(HEAD_CONTENTS MATCHES "ref") - # named branch - string(REPLACE "ref: " "" HEAD_REF "${HEAD_CONTENTS}") - if(EXISTS "@GIT_DIR@/${HEAD_REF}") - configure_file("@GIT_DIR@/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY) - else() - configure_file("@GIT_DIR@/packed-refs" "@GIT_DATA@/packed-refs" COPYONLY) - file(READ "@GIT_DATA@/packed-refs" PACKED_REFS) - if(${PACKED_REFS} MATCHES "([0-9a-z]*) ${HEAD_REF}") - set(HEAD_HASH "${CMAKE_MATCH_1}") - endif() - endif() -else() - # detached HEAD - configure_file("@GIT_DIR@/HEAD" "@GIT_DATA@/head-ref" COPYONLY) -endif() - -if(NOT HEAD_HASH) - file(READ "@GIT_DATA@/head-ref" HEAD_HASH LIMIT 1024) - string(STRIP "${HEAD_HASH}" HEAD_HASH) -endif() diff --git a/build/cmake/GetGitTimestamp.cmake b/build/cmake/GetGitTimestamp.cmake deleted file mode 100644 index f3c878844..000000000 --- a/build/cmake/GetGitTimestamp.cmake +++ /dev/null @@ -1,38 +0,0 @@ -function(get_git_timestamp _var) - if(NOT GIT_FOUND) - find_package(Git QUIET) - endif() - get_git_head_revision(refspec hash) - if(NOT GIT_FOUND) - set(${_var} "GIT-NOTFOUND" PARENT_SCOPE) - return() - endif() - - execute_process(COMMAND - git log --max-count=1 --date=iso --format=%cd ${hash} - WORKING_DIRECTORY - "${CMAKE_CURRENT_SOURCE_DIR}" - RESULT_VARIABLE - res - OUTPUT_VARIABLE - out - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT res EQUAL 0) - set(out "${out}-${res}-NOTFOUND") - endif() - - execute_process(COMMAND - date -d ${out} -u +%Y%m%d%H%M%S - WORKING_DIRECTORY - "${CMAKE_CURRENT_SOURCE_DIR}" - RESULT_VARIABLE - res - OUTPUT_VARIABLE - out - OUTPUT_STRIP_TRAILING_WHITESPACE) - if(NOT res EQUAL 0) - set(out "${out}-${res}-NOTFOUND") - endif() - - set(${_var} "${out}" PARENT_SCOPE) -endfunction() diff --git a/build/cmake/clean-download-archives.cmake b/build/cmake/clean-download-archives.cmake deleted file mode 100644 index bdbe6b12a..000000000 --- a/build/cmake/clean-download-archives.cmake +++ /dev/null @@ -1,4 +0,0 @@ -add_custom_target( - clean-download-archives - COMMAND rm -f ${WORK_DIR}/src/*.gz ${WORK_DIR}/src/*.bz2 ${WORK_DIR}/src/*.tgz -) diff --git a/build/cmake/core-admin-ui.cmake b/build/cmake/core-admin-ui.cmake deleted file mode 100644 index 3ebdb6ecd..000000000 --- a/build/cmake/core-admin-ui.cmake +++ /dev/null @@ -1,71 +0,0 @@ -include(${CMAKE_SOURCE_DIR}/build/cmake/dev/nodejs.cmake) - -file(GLOB_RECURSE admin_ui_files - ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.hbs - ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.html - ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.js - ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/app/*.scss - ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/config/*.js - ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/lib/*.js -) -add_custom_command( - OUTPUT - ${STAMP_DIR}/core-admin-ui-build-dir - ${CORE_BUILD_DIR}/tmp/admin-ui-build/package.json - DEPENDS - ${admin_ui_files} - ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/ember-cli-build.js - ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/package.json - ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/yarn.lock - COMMAND mkdir -p ${CORE_BUILD_DIR}/tmp/admin-ui-build - COMMAND rsync -a -v --delete-after "--filter=:- ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/.gitignore" --exclude=/dist ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/ ${CORE_BUILD_DIR}/tmp/admin-ui-build/ - COMMAND touch ${STAMP_DIR}/core-admin-ui-build-dir -) - -add_custom_command( - OUTPUT - ${STAMP_DIR}/core-admin-ui-yarn-install - ${CORE_BUILD_DIR}/tmp/admin-ui-build/node_modules - DEPENDS - yarn - ${STAMP_DIR}/core-admin-ui-build-dir - COMMAND cd ${CORE_BUILD_DIR}/tmp/admin-ui-build && env PATH=${DEV_INSTALL_PREFIX}/bin:$ENV{PATH} yarn install --frozen-lockfile - # In the CI environment, the "node-sass/vendor" directory seems to sometimes - # go away. A bit of a hack, but try to workaround this by forcing node-sass - # to be reinstalled if the vendor dir is missing. - # - # See: - # https://github.com/yarnpkg/yarn/issues/1981 - # https://github.com/yarnpkg/yarn/issues/1832 - # https://github.com/sass/node-sass/issues/1579 - COMMAND cd ${CORE_BUILD_DIR}/tmp/admin-ui-build && test -d node_modules/node-sass && test -d node_modules/node-sass/vendor || env PATH=${DEV_INSTALL_PREFIX}/bin:$ENV{PATH} yarn add node-sass --force - COMMAND touch ${STAMP_DIR}/core-admin-ui-yarn-install -) - -add_custom_command( - OUTPUT - ${STAMP_DIR}/core-admin-ui-build - ${CORE_BUILD_DIR}/tmp/admin-ui-build/dist - DEPENDS - ${STAMP_DIR}/core-admin-ui-build-dir - ${STAMP_DIR}/core-admin-ui-yarn-install - COMMAND cd ${CORE_BUILD_DIR}/tmp/admin-ui-build && rm -rf ./dist && env PATH=${DEV_INSTALL_PREFIX}/bin:$ENV{PATH} ./node_modules/.bin/ember build --environment=production --output-path=./dist - COMMAND touch ${STAMP_DIR}/core-admin-ui-build -) - -# Normally we perform the yarn installs out-of-source (so the build takes place -# entirely out of source), but if testing/development is enabled for this -# build, then also create a local symlink within the source. This then allows -# for easier interactions with the application. -if(ENABLE_TEST_DEPENDENCIES) - add_custom_command( - OUTPUT ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/node_modules - DEPENDS - ${STAMP_DIR}/core-admin-ui-yarn-install - ${CORE_BUILD_DIR}/tmp/admin-ui-build/node_modules - COMMAND rm -rf ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/node_modules - COMMAND ln -snf ${CORE_BUILD_DIR}/tmp/admin-ui-build/node_modules ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/node_modules - COMMAND touch -h ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/node_modules - ) - add_custom_target(core-admin-ui-local-yarn ALL DEPENDS ${CMAKE_SOURCE_DIR}/src/api-umbrella/admin-ui/node_modules) -endif() diff --git a/build/cmake/core-lua-deps.cmake b/build/cmake/core-lua-deps.cmake deleted file mode 100644 index b59ea8ee6..000000000 --- a/build/cmake/core-lua-deps.cmake +++ /dev/null @@ -1,100 +0,0 @@ -include(${CMAKE_SOURCE_DIR}/build/cmake/luarocks_install.cmake) - -# LuaRock app dependencies -luarocks_install(argparse ${LUAROCK_ARGPARSE_VERSION} ${LUAROCK_ARGPARSE_HASH}) -luarocks_install(inspect ${LUAROCK_INSPECT_VERSION} ${LUAROCK_INSPECT_HASH}) -luarocks_install(libcidr-ffi ${LUAROCK_LIBCIDR_VERSION} ${LUAROCK_LIBCIDR_HASH} CIDR_DIR=${LUA_PREFIX} libcidr) -luarocks_install(lua-cmsgpack ${LUAROCK_CMSGPACK_VERSION} ${LUAROCK_CMSGPACK_HASH}) -luarocks_install(lua-iconv ${LUAROCK_ICONV_VERSION} ${LUAROCK_ICONV_HASH}) -luarocks_install(lua-resty-auto-ssl ${LUAROCK_RESTY_AUTO_SSL_VERSION} ${LUAROCK_RESTY_AUTO_SSL_HASH}) -luarocks_install(lua-resty-http ${LUAROCK_RESTY_HTTP_VERSION} ${LUAROCK_RESTY_HTTP_HASH}) -luarocks_install(lua-resty-uuid ${LUAROCK_RESTY_UUID_VERSION} ${LUAROCK_RESTY_UUID_HASH}) -luarocks_install(luaposix ${LUAROCK_LUAPOSIX_VERSION} ${LUAROCK_LUAPOSIX_HASH}) -luarocks_install(luatz ${LUAROCK_LUATZ_VERSION} ${LUAROCK_LUATZ_HASH}) -luarocks_install(lustache ${LUAROCK_LUSTACHE_VERSION} ${LUAROCK_LUSTACHE_HASH}) -luarocks_install(lyaml ${LUAROCK_LYAML_VERSION} ${LUAROCK_LYAML_HASH}) -luarocks_install(penlight ${LUAROCK_PENLIGHT_VERSION} ${LUAROCK_PENLIGHT_HASH}) - -# Other Lua app dependencies (non-luarocks) -ExternalProject_Add( - lua_luasocket_url - DEPENDS luarocks - URL https://github.com/diegonehab/luasocket/archive/${LUA_LUASOCKET_VERSION}.tar.gz - URL_HASH MD5=${LUA_LUASOCKET_HASH} - # Just install the URL parsing library from luasocket (rather than the whole - # luarocks, since we don't need the other parts, and the luarock is somewhat - # outdated). In order to just install this one file, patch it to work without - # the base luasocket library present (it doesn't actually use the base stuff - # for anything). - PATCH_COMMAND sed -i -e "s%local socket = require.*%local socket = {}%" src/url.lua - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND install -D -m 644 /src/url.lua ${VENDOR_LUA_DIR}/socket/url.lua -) - -ExternalProject_Add( - lua_resty_dns_cache - DEPENDS luarocks - URL https://github.com/hamishforbes/lua-resty-dns-cache/archive/${LUA_RESTY_DNS_CACHE_VERSION}.tar.gz - URL_HASH MD5=${LUA_RESTY_DNS_CACHE_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND install -D -m 644 /lib/resty/dns/cache.lua ${VENDOR_LUA_DIR}/resty/dns/cache.lua -) - -ExternalProject_Add( - lua_resty_logger_socket - DEPENDS luarocks - URL https://github.com/cloudflare/lua-resty-logger-socket/archive/${LUA_RESTY_LOGGER_SOCKET_VERSION}.tar.gz - URL_HASH MD5=${LUA_RESTY_LOGGER_SOCKET_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND install -D -m 644 /lib/resty/logger/socket.lua ${VENDOR_LUA_DIR}/resty/logger/socket.lua -) - -ExternalProject_Add( - lua_resty_shcache - DEPENDS luarocks - URL https://github.com/cloudflare/lua-resty-shcache/archive/${LUA_RESTY_SHCACHE_VERSION}.tar.gz - URL_HASH MD5=${LUA_RESTY_SHCACHE_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND install -D -m 644 /shcache.lua ${VENDOR_LUA_DIR}/shcache.lua -) - -set( - LUA_DEPS - lua_luasocket_url - lua_resty_dns_cache - lua_resty_logger_socket - lua_resty_shcache - luarock_argparse - luarock_inspect - luarock_libcidr-ffi - luarock_lua-cmsgpack - luarock_lua-iconv - luarock_lua-resty-auto-ssl - luarock_lua-resty-http - luarock_lua-resty-uuid - luarock_luaposix - luarock_luatz - luarock_lustache - luarock_lyaml - luarock_penlight -) - -# Also depend on the internal stamp files used by ExternalProject_Add, since -# add_custom_command seems to require files to properly work when updates -# occur (we can't just specify the ExternalProject_Add target names or -# updates aren't detected). But we still need to depend on the project names -# directly for the initial install dependency ordering. -foreach(LUA_DEP ${LUA_DEPS}) - list(APPEND LUA_DEPS_DEPENDS ${LUA_DEP}) - list(APPEND LUA_DEPS_DEPENDS ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${LUA_DEP}-complete) -endforeach() - -add_custom_command( - OUTPUT ${STAMP_DIR}/core-lua-deps - DEPENDS ${LUA_DEPS_DEPENDS} - COMMAND touch ${STAMP_DIR}/core-lua-deps -) diff --git a/build/cmake/core-web-app.cmake b/build/cmake/core-web-app.cmake deleted file mode 100644 index 7f6bcf439..000000000 --- a/build/cmake/core-web-app.cmake +++ /dev/null @@ -1,49 +0,0 @@ -add_custom_command( - OUTPUT - ${STAMP_DIR}/core-web-app-bundle - ${WORK_DIR}/src/web-app/.bundle - ${VENDOR_DIR}/bundle - DEPENDS - bundler - ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Gemfile - ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Gemfile.lock - COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} BUNDLE_GEMFILE=${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Gemfile BUNDLE_APP_CONFIG=${WORK_DIR}/src/web-app/.bundle bundle install --clean --path=${VENDOR_DIR}/bundle - COMMAND touch -c ${WORK_DIR}/src/web-app/.bundle - COMMAND touch -c ${VENDOR_DIR}/bundle - COMMAND touch ${STAMP_DIR}/core-web-app-bundle -) - -file(GLOB_RECURSE web_asset_files - ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/app/assets/*.css - ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/app/assets/*.scss - ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/app/assets/*.erb - ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/app/assets/*.js -) -add_custom_command( - OUTPUT ${STAMP_DIR}/core-web-app-precompile - DEPENDS - ${STAMP_DIR}/core-web-app-bundle - ${web_asset_files} - ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/config/initializers/assets.rb - COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} BUNDLE_GEMFILE=${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Gemfile BUNDLE_APP_CONFIG=${WORK_DIR}/src/web-app/.bundle RAILS_TMP_PATH=${CORE_BUILD_DIR}/tmp/web-app-tmp RAILS_PUBLIC_PATH=${CORE_BUILD_DIR}/tmp/web-app-build RAILS_ENV=production RAILS_SECRET_TOKEN=temp RAILS_ASSETS_PRECOMPILE=true bundle exec rake -f ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Rakefile assets:clobber assets:precompile - COMMAND touch ${STAMP_DIR}/core-web-app-precompile -) - -# Normally we perform the bundle out-of-source (so the build takes place -# entirely out of source), but if testing/development is enabled for this -# build, then also create a local ".bundle/config" item within the source. This -# then allows for gems to be found when interacting with the local source -# version of the app. -if(ENABLE_TEST_DEPENDENCIES) - add_custom_command( - OUTPUT ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/.bundle/config - DEPENDS - ${STAMP_DIR}/core-web-app-bundle - ${WORK_DIR}/src/web-app/.bundle - ${VENDOR_DIR}/bundle - COMMAND rm -rf ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/.bundle - COMMAND ln -snf ${WORK_DIR}/src/web-app/.bundle ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/.bundle - COMMAND touch -c ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/.bundle/config - ) - add_custom_target(core-web-app-local-bundle ALL DEPENDS ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/.bundle/config) -endif() diff --git a/build/cmake/core.cmake b/build/cmake/core.cmake deleted file mode 100644 index 959f10e14..000000000 --- a/build/cmake/core.cmake +++ /dev/null @@ -1,142 +0,0 @@ -set(CORE_BUILD_DIR ${WORK_DIR}/src/api-umbrella-core) - -include(${CMAKE_SOURCE_DIR}/build/cmake/core-lua-deps.cmake) -include(${CMAKE_SOURCE_DIR}/build/cmake/core-web-app.cmake) -include(${CMAKE_SOURCE_DIR}/build/cmake/core-admin-ui.cmake) - -# Copy the vendored libraries into the shared build directory. -add_custom_command( - OUTPUT ${CORE_BUILD_DIR}/shared/vendor - DEPENDS - ${STAMP_DIR}/core-web-app-bundle - ${STAMP_DIR}/core-lua-deps - COMMAND mkdir -p ${CORE_BUILD_DIR}/shared/vendor - COMMAND rsync -a --delete-after ${VENDOR_DIR}/ ${CORE_BUILD_DIR}/shared/vendor/ - COMMAND touch -c ${CORE_BUILD_DIR}/shared/vendor -) - -# Copy the code into a build release directory. -file(GLOB_RECURSE core_files - ${CMAKE_SOURCE_DIR}/bin/* - ${CMAKE_SOURCE_DIR}/config/* - ${CMAKE_SOURCE_DIR}/templates/* - ${CMAKE_SOURCE_DIR}/src/api-umbrella/web-app/Gemfile* -) -add_custom_command( - OUTPUT ${STAMP_DIR}/core-build-release-dir - DEPENDS ${core_files} - COMMAND mkdir -p ${CORE_BUILD_DIR}/releases/0 - COMMAND rsync -a --delete-after --delete-excluded "--filter=:- ${CMAKE_SOURCE_DIR}/.gitignore" --include=/templates/etc/perp/.boot --exclude=.* --exclude=/templates/etc/test-env* --exclude=/templates/etc/perp/test-env* --exclude=/src/api-umbrella/web-app/spec --exclude=/src/api-umbrella/web-app/app/assets --exclude=/src/api-umbrella/hadoop-analytics --include=/bin/*** --include=/config/*** --include=/LICENSE.txt --include=/templates/*** --include=/src/*** --exclude=* ${CMAKE_SOURCE_DIR}/ ${CORE_BUILD_DIR}/releases/0/ - COMMAND touch ${STAMP_DIR}/core-build-release-dir -) - -add_custom_command( - OUTPUT - ${STAMP_DIR}/core-build-install-dist - ${CORE_BUILD_DIR}/releases/0/build/dist/web-app-assets - ${CORE_BUILD_DIR}/releases/0/build/dist/admin-ui - DEPENDS - ${STAMP_DIR}/core-admin-ui-build - ${STAMP_DIR}/core-web-app-precompile - ${STAMP_DIR}/core-build-release-dir - COMMAND mkdir -p ${CORE_BUILD_DIR}/releases/0/build/dist/web-app-assets - COMMAND rsync -a --delete-after ${CORE_BUILD_DIR}/tmp/web-app-build/web-assets/ ${CORE_BUILD_DIR}/releases/0/build/dist/web-app-assets/web-assets/ - COMMAND rsync -a --delete-after ${CORE_BUILD_DIR}/tmp/admin-ui-build/dist/ ${CORE_BUILD_DIR}/releases/0/build/dist/admin-ui/ - COMMAND touch ${STAMP_DIR}/core-build-install-dist -) - -# Create a symlink to the latest release. -add_custom_command( - OUTPUT ${STAMP_DIR}/core-build-current-symlink - DEPENDS ${STAMP_DIR}/core-build-release-dir - WORKING_DIRECTORY ${CORE_BUILD_DIR} - COMMAND ln -snf releases/0 ./current - COMMAND touch ${STAMP_DIR}/core-build-current-symlink -) - -# Create a symlink to the shared vendor directory within the release. -add_custom_command( - OUTPUT ${STAMP_DIR}/core-build-release-vendor-symlink - DEPENDS - ${STAMP_DIR}/core-build-release-dir - ${CORE_BUILD_DIR}/shared/vendor - WORKING_DIRECTORY ${CORE_BUILD_DIR}/releases/0 - COMMAND ln -snf ../../shared/vendor ./vendor - COMMAND touch ${STAMP_DIR}/core-build-release-vendor-symlink -) - -# Copy the gems into the build directory and cleanup for production use. -add_custom_command( - OUTPUT ${CORE_BUILD_DIR}/releases/0/src/api-umbrella/web-app/.bundle/config - DEPENDS - ${STAMP_DIR}/core-build-release-dir - ${STAMP_DIR}/core-build-release-vendor-symlink - WORKING_DIRECTORY ${CORE_BUILD_DIR}/releases/0/src/api-umbrella/web-app - # Disable all non-production gems and remove any old, unused gems. - COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} bundle install --path=../../../vendor/bundle --without=development test assets --clean --deployment - # Purge gem files we don't need to make for a lighter package distribution. - COMMAND cd ${CORE_BUILD_DIR}/shared/vendor/bundle && rm -rf ruby/*/cache ruby/*/gems/*/test* ruby/*/gems/*/spec ruby/*/bundler/gems/*/test* ruby/*/bundler/gems/*/spec ruby/*/bundler/gems/*/.git - COMMAND touch -c ${CORE_BUILD_DIR}/releases/0/src/api-umbrella/web-app/.bundle/config -) - -# -# Build the release dir. -# -add_custom_command( - OUTPUT ${STAMP_DIR}/core-build-release - DEPENDS - ${STAMP_DIR}/core-build-release-dir - ${STAMP_DIR}/core-build-release-vendor-symlink - ${STAMP_DIR}/core-build-current-symlink - ${CORE_BUILD_DIR}/releases/0/src/api-umbrella/web-app/.bundle/config - COMMAND touch ${STAMP_DIR}/core-build-release -) - -# Copy the built shared directory to the stage install path. -add_custom_command( - OUTPUT ${STAGE_EMBEDDED_DIR}/apps/core - DEPENDS ${CORE_BUILD_DIR}/shared/vendor - DEPENDS ${STAMP_DIR}/core-build-release - DEPENDS ${STAMP_DIR}/core-build-install-dist - COMMAND mkdir -p ${STAGE_EMBEDDED_DIR}/apps/core - COMMAND rsync -a --delete-after --delete-excluded --exclude=/tmp ${CORE_BUILD_DIR}/ ${STAGE_EMBEDDED_DIR}/apps/core/ - COMMAND touch -c ${STAGE_EMBEDDED_DIR}/apps/core -) - -# Create a symlink for the main "api-umbrella" binary. -add_custom_command( - OUTPUT ${STAMP_DIR}/core-api-umbrella-bin-symlink - DEPENDS ${STAGE_EMBEDDED_DIR}/apps/core - COMMAND mkdir -p ${STAGE_PREFIX_DIR}/bin - COMMAND cd ${STAGE_PREFIX_DIR}/bin && ln -snf ../embedded/apps/core/current/bin/api-umbrella ./api-umbrella - COMMAND touch ${STAMP_DIR}/core-api-umbrella-bin-symlink -) -add_custom_command( - OUTPUT ${STAMP_DIR}/core-api-umbrella-env-bin-symlink - DEPENDS ${STAGE_EMBEDDED_DIR}/apps/core - COMMAND mkdir -p ${STAGE_PREFIX_DIR}/bin - COMMAND cd ${STAGE_PREFIX_DIR}/bin && ln -snf ../embedded/apps/core/current/bin/api-umbrella-env ./api-umbrella-env - COMMAND touch ${STAMP_DIR}/core-api-umbrella-env-bin-symlink -) -add_custom_command( - OUTPUT ${STAMP_DIR}/core-api-umbrella-exec-bin-symlink - DEPENDS ${STAGE_EMBEDDED_DIR}/apps/core - COMMAND mkdir -p ${STAGE_PREFIX_DIR}/bin - COMMAND cd ${STAGE_PREFIX_DIR}/bin && ln -snf ../embedded/apps/core/current/bin/api-umbrella-exec ./api-umbrella-exec - COMMAND touch ${STAMP_DIR}/core-api-umbrella-exec-bin-symlink -) - -# -# Install the core app into the stage location. -# -add_custom_command( - OUTPUT ${STAMP_DIR}/core - DEPENDS - ${STAGE_EMBEDDED_DIR}/apps/core - ${STAMP_DIR}/core-api-umbrella-bin-symlink - ${STAMP_DIR}/core-api-umbrella-env-bin-symlink - ${STAMP_DIR}/core-api-umbrella-exec-bin-symlink - COMMAND touch ${STAMP_DIR}/core -) - -add_custom_target(core ALL DEPENDS ${STAMP_DIR}/core) diff --git a/build/cmake/dev/nodejs.cmake b/build/cmake/dev/nodejs.cmake deleted file mode 100644 index 057e2aac2..000000000 --- a/build/cmake/dev/nodejs.cmake +++ /dev/null @@ -1,22 +0,0 @@ -# NodeJS: For building admin-ui Ember app. -ExternalProject_Add( - nodejs - URL https://nodejs.org/dist/v${NODEJS_VERSION}/node-v${NODEJS_VERSION}-linux-x64.tar.xz - URL_HASH SHA256=${NODEJS_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND rsync -a -v / ${DEV_INSTALL_PREFIX}/ -) - -ExternalProject_Add( - yarn - DEPENDS nodejs - URL https://github.com/yarnpkg/yarn/releases/download/v${YARN_VERSION}/yarn-v${YARN_VERSION}.tar.gz - URL_HASH MD5=${YARN_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND rsync -a -v --delete / ${DEV_INSTALL_PREFIX}/yarn/ - COMMAND cd ${DEV_INSTALL_PREFIX}/bin && ln -snf ../yarn/bin/yarn ./yarn - # Remove the previous bin symlink that was necessary. - COMMAND rm -f ${DEV_INSTALL_PREFIX}/bin/yarn.js -) diff --git a/build/cmake/distclean.cmake b/build/cmake/distclean.cmake deleted file mode 100644 index 8e9030542..000000000 --- a/build/cmake/distclean.cmake +++ /dev/null @@ -1,4 +0,0 @@ -add_custom_target( - distclean - COMMAND ${CMAKE_SOURCE_DIR}/build/scripts/distclean -) diff --git a/build/cmake/hadoop-analytics/flume.cmake b/build/cmake/hadoop-analytics/flume.cmake deleted file mode 100644 index 1605204c3..000000000 --- a/build/cmake/hadoop-analytics/flume.cmake +++ /dev/null @@ -1,12 +0,0 @@ -find_package(Java 1.7 REQUIRED COMPONENTS Runtime) - -# Flume: Hadoop log buffering and writing -ExternalProject_Add( - flume - URL http://apache.cs.utah.edu/flume/${FLUME_VERSION}/apache-flume-${FLUME_VERSION}-bin.tar.gz - URL_HASH MD5=${FLUME_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND mkdir -p ${HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR}/flume - COMMAND rsync -a -v --exclude=/docs --exclude=/tools --delete-after --delete-excluded / ${HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR}/flume/ -) diff --git a/build/cmake/hadoop-analytics/kylin.cmake b/build/cmake/hadoop-analytics/kylin.cmake deleted file mode 100644 index 047e73510..000000000 --- a/build/cmake/hadoop-analytics/kylin.cmake +++ /dev/null @@ -1,12 +0,0 @@ -find_package(Java 1.7 REQUIRED COMPONENTS Runtime) - -# Kylin: Hadoop-based analytics database -ExternalProject_Add( - kylin - URL http://mirrors.sonic.net/apache/kylin/apache-kylin-${KYLIN_VERSION}/apache-kylin-${KYLIN_VERSION}-bin.tar.gz - URL_HASH MD5=${KYLIN_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND mkdir -p ${HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR}/kylin - COMMAND rsync -a -v --exclude=/sample_cube --delete-after --delete-excluded / ${HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR}/kylin/ -) diff --git a/build/cmake/hadoop-analytics/presto.cmake b/build/cmake/hadoop-analytics/presto.cmake deleted file mode 100644 index 8f5db3cbf..000000000 --- a/build/cmake/hadoop-analytics/presto.cmake +++ /dev/null @@ -1,12 +0,0 @@ -find_package(Java 1.7 REQUIRED COMPONENTS Runtime) - -# Presto: ANSI-SQL queries against Hadoop Hive tables. -ExternalProject_Add( - presto - URL https://repo1.maven.org/maven2/com/facebook/presto/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz - URL_HASH MD5=${PRESTO_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND mkdir -p ${HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR}/presto - COMMAND rsync -a -v --include=/plugin/hive-hadoop2 --include=/plugin/jmx --exclude=/plugin/* --delete-after --delete-excluded / ${HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR}/presto/ -) diff --git a/build/cmake/hadoop-analytics/processor.cmake b/build/cmake/hadoop-analytics/processor.cmake deleted file mode 100644 index 9f0ec9471..000000000 --- a/build/cmake/hadoop-analytics/processor.cmake +++ /dev/null @@ -1,25 +0,0 @@ -find_package(Java 1.7 REQUIRED COMPONENTS Runtime) -find_package(Java 1.7 REQUIRED COMPONENTS Development) - -ExternalProject_Add( - maven - URL http://apache.mirrors.ionfish.org/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz - URL_HASH MD5=${MAVEN_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" -) -ExternalProject_Get_Property(maven SOURCE_DIR) -set(MAVEN_SOURCE_DIR ${SOURCE_DIR}) - -add_custom_command( - OUTPUT ${STAMP_DIR}/hadoop-analytics-processor - DEPENDS - maven - COMMAND mkdir -p ${WORK_DIR}/src/hadoop-analytics - COMMAND env PATH=${MAVEN_SOURCE_DIR}/bin:$ENV{PATH} mvn -f ${CMAKE_SOURCE_DIR}/src/api-umbrella/hadoop-analytics/pom.xml clean package -DbuildDir=${WORK_DIR}/src/hadoop-analytics - COMMAND mkdir -p ${HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR}/hadoop-analytics - COMMAND cp ${WORK_DIR}/src/hadoop-analytics/processor/processor-0.0.1-SNAPSHOT.jar ${HADOOP_ANALYTICS_STAGE_EMBEDDED_DIR}/hadoop-analytics/processor.jar - COMMAND touch ${STAMP_DIR}/hadoop-analytics-processor -) -add_custom_target(hadoop-analytics-processor ALL DEPENDS ${STAMP_DIR}/hadoop-analytics-processor) diff --git a/build/cmake/install.cmake b/build/cmake/install.cmake deleted file mode 100644 index 9c0489ee4..000000000 --- a/build/cmake/install.cmake +++ /dev/null @@ -1,113 +0,0 @@ -install( - DIRECTORY ${STAGE_PREFIX_DIR}/ - DESTINATION ${CMAKE_INSTALL_PREFIX} - USE_SOURCE_PERMISSIONS - COMPONENT core -) -install( - PROGRAMS ${CMAKE_SOURCE_DIR}/build/package/files/etc/init.d/api-umbrella - DESTINATION /etc/init.d - COMPONENT core -) -install( - FILES ${CMAKE_SOURCE_DIR}/build/package/files/etc/logrotate.d/api-umbrella - DESTINATION /etc/logrotate.d - COMPONENT core -) -install( - FILES ${CMAKE_SOURCE_DIR}/build/package/files/etc/sudoers.d/api-umbrella - DESTINATION /etc/sudoers.d - PERMISSIONS OWNER_READ GROUP_READ - COMPONENT core -) - -# If /etc/api-umbrella/api-umbrella.yml doesn't exist, install it. -# -# If /etc/api-umbrella/api-umbrella.yml does exist, install the default version -# to api-umbrella.yml.default (so it's available for reference, but we don't -# overwrite any local changes). -# -# The CODE block is so that this conditional is deferred until install time -# (rather than when cmake builds the makefile). See: -# https://cmake.org/Bug/view.php?id=12646 -install( - CODE " - if(NOT EXISTS \$ENV{DESTDIR}/etc/api-umbrella/api-umbrella.yml) - file(INSTALL ${CMAKE_SOURCE_DIR}/build/package/files/etc/api-umbrella/api-umbrella.yml DESTINATION /etc/api-umbrella) - else() - message(STATUS \"Skipping: \$ENV{DESTDIR}/etc/api-umbrella/api-umbrella.yml\") - file(INSTALL ${CMAKE_SOURCE_DIR}/build/package/files/etc/api-umbrella/api-umbrella.yml DESTINATION /etc/api-umbrella RENAME api-umbrella.yml.default) - endif() - " - COMPONENT core -) - -install( - CODE " - message(STATUS \"Directories: \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/apps/core/releases/${RELEASE_TIMESTAMP}\") - execute_process( - WORKING_DIRECTORY \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/apps/core - # Multiple commands in execute_process are executed in parallel - # (http://public.kitware.com/pipermail/cmake/2016-March/063076.html). Since - # the order of these matter, use a shell wrapper. - COMMAND sh -c \"rm -rf releases/${RELEASE_TIMESTAMP} && mv releases/0 releases/${RELEASE_TIMESTAMP} && ln -snf releases/${RELEASE_TIMESTAMP} ./current\" - ) - message(STATUS \"Directories: \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/apps/static-site/releases/${RELEASE_TIMESTAMP}\") - execute_process( - WORKING_DIRECTORY \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/apps/static-site - # Multiple commands in execute_process are executed in parallel - # (http://public.kitware.com/pipermail/cmake/2016-March/063076.html). Since - # the order of these matter, use a shell wrapper. - COMMAND sh -c \"rm -rf releases/${RELEASE_TIMESTAMP} && mv releases/0 releases/${RELEASE_TIMESTAMP} && ln -snf releases/${RELEASE_TIMESTAMP} ./current\" - ) - message(STATUS \"Directories: \$ENV{DESTDIR}/usr/bin \$ENV{DESTDIR}/var/log \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/etc \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/db \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/log \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/run \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/tmp\") - execute_process( - COMMAND mkdir -p \$ENV{DESTDIR}/usr/bin \$ENV{DESTDIR}/var/log \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/etc \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/db \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/log \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/run \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/tmp - ) - message(STATUS \"Installing: \$ENV{DESTDIR}/usr/bin/api-umbrella\") - execute_process( - WORKING_DIRECTORY \$ENV{DESTDIR}/usr/bin - COMMAND ln -snf ../..${CMAKE_INSTALL_PREFIX}/bin/api-umbrella ./api-umbrella - ) - message(STATUS \"Installing: \$ENV{DESTDIR}/var/log/api-umbrella\") - execute_process( - WORKING_DIRECTORY \$ENV{DESTDIR}/var/log - COMMAND ln -snf ../..${CMAKE_INSTALL_PREFIX}/var/log ./api-umbrella - ) - message(STATUS \"Replacing: \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/openresty/luajit/bin/luarocks-5.1 \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/openresty/luajit/bin/luarocks-admin-5.1 \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/openresty/luajit/share/lua/5.1/luarocks/site_config.lua\") - execute_process( - COMMAND sed -i \"s#${STAGE_DIR}##g\" \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/openresty/luajit/bin/luarocks-5.1 \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/openresty/luajit/bin/luarocks-admin-5.1 \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/embedded/openresty/luajit/share/lua/5.1/luarocks/site_config.lua - ) - message(STATUS \"Permissions: \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/tmp\") - execute_process( - COMMAND chmod 1777 \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/var/tmp - ) - " - COMPONENT core -) - -if(ENABLE_HADOOP_ANALYTICS) - install( - DIRECTORY ${HADOOP_ANALYTICS_STAGE_PREFIX_DIR}/ - DESTINATION ${CMAKE_INSTALL_PREFIX} - USE_SOURCE_PERMISSIONS - COMPONENT hadoop-analytics - ) -endif() - -add_custom_target( - install-core - COMMAND ${CMAKE_COMMAND} -D CMAKE_INSTALL_COMPONENT=core -P ${CMAKE_BINARY_DIR}/cmake_install.cmake -) - -if(ENABLE_HADOOP_ANALYTICS) - add_custom_target( - install-hadoop-analytics - COMMAND ${CMAKE_COMMAND} -D CMAKE_INSTALL_COMPONENT=hadoop-analytics -P ${CMAKE_BINARY_DIR}/cmake_install.cmake - ) -endif() - -add_custom_target( - after-install - COMMAND ${CMAKE_SOURCE_DIR}/build/package/scripts/after-install 1 -) diff --git a/build/cmake/libcidr.cmake b/build/cmake/libcidr.cmake deleted file mode 100644 index 45cc6cecd..000000000 --- a/build/cmake/libcidr.cmake +++ /dev/null @@ -1,10 +0,0 @@ -# libcidr: CIDR IP calculations for libcidr-ffi LuaRock -ExternalProject_Add( - libcidr - URL https://www.over-yonder.net/~fullermd/projects/libcidr/libcidr-${LIBCIDR_VERSION}.tar.xz - URL_HASH MD5=${LIBCIDR_HASH} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND make PREFIX=${INSTALL_PREFIX_EMBEDDED} - INSTALL_COMMAND make install NO_DOCS=1 NO_EXAMPLES=1 PREFIX=${INSTALL_PREFIX_EMBEDDED} DESTDIR=${STAGE_DIR} -) diff --git a/build/cmake/libgeoip.cmake b/build/cmake/libgeoip.cmake deleted file mode 100644 index 490d6941b..000000000 --- a/build/cmake/libgeoip.cmake +++ /dev/null @@ -1,27 +0,0 @@ -# libgeoip & GeoLiteCityv6.dat: GeoIP locations -list(APPEND LIBGEOIP_CONFIGURE_CMD env) -list(APPEND LIBGEOIP_CONFIGURE_CMD /configure) -list(APPEND LIBGEOIP_CONFIGURE_CMD --prefix=${INSTALL_PREFIX_EMBEDDED}) - -ExternalProject_Add( - libgeoip - URL https://github.com/maxmind/geoip-api-c/releases/download/v${LIBGEOIP_VERSION}/GeoIP-${LIBGEOIP_VERSION}.tar.gz - URL_HASH MD5=${LIBGEOIP_HASH} - CONFIGURE_COMMAND ${LIBGEOIP_CONFIGURE_CMD} - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} - COMMAND find ${STAGE_EMBEDDED_DIR}/bin/ -name geoiplookup* -exec chrpath -d {} $ -) - -ExternalProject_Add( - # Make the project name dynamic based on the current date. This forces a - # re-download once per day. This helps ensure development and CI environments - # are using fresh GeoIP data files without downloading on each run. - geolitecity-${RELEASE_DATE} - URL https://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz - DOWNLOAD_NO_EXTRACT 1 - # Since we re-download every day as a separate project name, this cleans up - # any old downloads in the work directory. - CONFIGURE_COMMAND find ${CMAKE_BINARY_DIR}/${EP_BASE} -maxdepth 2 -name geolitecity* -not -name geolitecity-${RELEASE_DATE}* -print -exec rm -rf {} $ - BUILD_COMMAND gunzip -c > /GeoLiteCityv6.dat - INSTALL_COMMAND install -D -m 644 /GeoLiteCityv6.dat ${STAGE_EMBEDDED_DIR}/var/db/geoip/city-v6.dat -) diff --git a/build/cmake/luarocks.cmake b/build/cmake/luarocks.cmake deleted file mode 100644 index ae8e0adee..000000000 --- a/build/cmake/luarocks.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# LuaRocks: Lua dependency management -ExternalProject_Add( - luarocks - DEPENDS openresty - URL http://luarocks.org/releases/luarocks-${LUAROCKS_VERSION}.tar.gz - URL_HASH MD5=${LUAROCKS_HASH} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND /configure --prefix=${INSTALL_PREFIX_EMBEDDED}/openresty/luajit --with-lua=${STAGE_EMBEDDED_DIR}/openresty/luajit --with-lua-include=${STAGE_EMBEDDED_DIR}/openresty/luajit/include/luajit-2.1 --lua-suffix=jit - BUILD_COMMAND make build - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} - COMMAND cd ${STAGE_EMBEDDED_DIR}/bin && ln -snf ../openresty/luajit/bin/luarocks ./luarocks - COMMAND rm -rf ${VENDOR_DIR}/share/lua ${VENDOR_DIR}/lib/luarocks - COMMAND rm -rf ${TEST_VENDOR_DIR}/share/lua ${TEST_VENDOR_DIR}/lib/luarocks -) diff --git a/build/cmake/luarocks_install.cmake b/build/cmake/luarocks_install.cmake deleted file mode 100644 index 66a5a34de..000000000 --- a/build/cmake/luarocks_install.cmake +++ /dev/null @@ -1,21 +0,0 @@ -function(_luarocks_install tree_dir package version hash) - ExternalProject_Add( - luarock_${package} - DEPENDS luarocks ${ARGV5} - URL https://luarocks.org/${package}-${version}.rockspec - URL_HASH MD5=${hash} - DOWNLOAD_NO_EXTRACT 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND ${LUAROCKS_CMD} --tree=${tree_dir} install ${package} ${version} ${ARGV4} - COMMAND find ${tree_dir} -name *.so -exec chrpath -d {} $ - ) -endfunction() - -function(luarocks_install package version hash) - _luarocks_install(${VENDOR_DIR} ${package} ${version} ${hash} ${ARGV3} ${ARGV4}) -endfunction() - -function(test_luarocks_install package version hash) - _luarocks_install(${TEST_VENDOR_DIR} ${package} ${version} ${hash} ${ARGV3} ${ARGV4}) -endfunction() diff --git a/build/cmake/mora.cmake b/build/cmake/mora.cmake deleted file mode 100644 index 25812b366..000000000 --- a/build/cmake/mora.cmake +++ /dev/null @@ -1,24 +0,0 @@ -# Mora: HTTP API for MongoDB (allowing OpenResty connectivity) -# Built with Go -ExternalProject_Add( - golang - URL https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-amd64.tar.gz - URL_HASH SHA256=${GOLANG_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" -) -ExternalProject_Get_Property(golang SOURCE_DIR) -set(GOLANG_SOURCE_DIR ${SOURCE_DIR}) - -ExternalProject_Add( - mora - DEPENDS golang - URL https://github.com/emicklei/mora/archive/${MORA_VERSION}.tar.gz - URL_HASH MD5=${MORA_HASH} - SOURCE_DIR ${WORK_DIR}/gocode/src/github.com/emicklei/mora - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND env PATH=${GOLANG_SOURCE_DIR}/bin:${WORK_DIR}/gocode/bin:$ENV{PATH} GOPATH=${WORK_DIR}/gocode GOROOT=${GOLANG_SOURCE_DIR} go install - INSTALL_COMMAND install -D -m 755 ${WORK_DIR}/gocode/bin/mora ${STAGE_EMBEDDED_DIR}/bin/mora -) diff --git a/build/cmake/openresty.cmake b/build/cmake/openresty.cmake deleted file mode 100644 index 9331f613b..000000000 --- a/build/cmake/openresty.cmake +++ /dev/null @@ -1,92 +0,0 @@ -# OpenResty and nginx plugins - -# ngx_dyups: Dynamic upstream handling for handling DNS changes -ExternalProject_Add( - ngx_dyups - URL https://github.com/yzprofile/ngx_http_dyups_module/archive/${NGX_DYUPS_VERSION}.tar.gz - URL_HASH MD5=${NGX_DYUPS_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" -) -ExternalProject_Get_Property(ngx_dyups SOURCE_DIR) -set(NGX_DYUPS_SOURCE_DIR ${SOURCE_DIR}) - -# ngx_txid: Generate unique request IDs -ExternalProject_Add( - ngx_txid - URL https://github.com/streadway/ngx_txid/archive/${NGX_TXID_VERSION}.tar.gz - URL_HASH MD5=${NGX_TXID_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" -) -ExternalProject_Get_Property(ngx_txid SOURCE_DIR) -set(NGX_TXID_SOURCE_DIR ${SOURCE_DIR}) - -# Pull in newer version of PCRE (8.20+) for OpenResty to enable PCRE JIT. -ExternalProject_Add( - pcre - URL https://ftp.pcre.org/pub/pcre/pcre-${PCRE_VERSION}.tar.bz2 - URL_HASH MD5=${PCRE_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" -) -ExternalProject_Get_Property(pcre SOURCE_DIR) -set(PCRE_SOURCE_DIR ${SOURCE_DIR}) - -# OpenResty's ssl_certificate_by_lua functionality requires OpenSSL 1.0.2e+ -ExternalProject_Add( - openssl - URL https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz - URL_HASH SHA256=${OPENSSL_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" -) -ExternalProject_Get_Property(openssl SOURCE_DIR) -set(OPENSSL_SOURCE_DIR ${SOURCE_DIR}) - -list(APPEND OPENRESTY_CONFIGURE_CMD /configure) -list(APPEND OPENRESTY_CONFIGURE_CMD --prefix=${INSTALL_PREFIX_EMBEDDED}/openresty) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-cc-opt=-I${STAGE_EMBEDDED_DIR}/include) -if(ENABLE_TEST_DEPENDENCIES) - list(APPEND OPENRESTY_CONFIGURE_CMD "--with-ld-opt=-L${STAGE_EMBEDDED_DIR}/lib") -else() - list(APPEND OPENRESTY_CONFIGURE_CMD "--with-ld-opt=-L${STAGE_EMBEDDED_DIR}/lib") -endif() -list(APPEND OPENRESTY_CONFIGURE_CMD --error-log-path=stderr) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-ipv6) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-openssl=${OPENSSL_SOURCE_DIR}) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-pcre=${PCRE_SOURCE_DIR}) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-pcre-opt=-g) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-pcre-conf-opt=--enable-unicode-properties) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-pcre-jit) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-http_geoip_module) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-http_gunzip_module) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-http_gzip_static_module) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-http_realip_module) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-http_ssl_module) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-http_v2_module) -list(APPEND OPENRESTY_CONFIGURE_CMD --with-http_stub_status_module) -list(APPEND OPENRESTY_CONFIGURE_CMD --add-module=${NGX_DYUPS_SOURCE_DIR}) -list(APPEND OPENRESTY_CONFIGURE_CMD --add-module=${NGX_TXID_SOURCE_DIR}) - -ExternalProject_Add( - openresty - DEPENDS libgeoip ngx_dyups ngx_txid openssl pcre - URL https://openresty.org/download/openresty-${OPENRESTY_VERSION}.tar.gz - URL_HASH MD5=${OPENRESTY_HASH} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND ${OPENRESTY_CONFIGURE_CMD} - # Wipe the .openssl directory inside the openssl dir, or else openresty - # will fail to build on rebuilds: https://trac.nginx.org/nginx/ticket/583 - BUILD_COMMAND COMMAND cd ${OPENSSL_SOURCE_DIR} && rm -rf .openssl - COMMAND make - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} - COMMAND chrpath -d ${STAGE_EMBEDDED_DIR}/openresty/nginx/sbin/nginx - COMMAND cd ${STAGE_EMBEDDED_DIR}/bin && ln -snf ../openresty/bin/resty ./resty - COMMAND cd ${STAGE_EMBEDDED_DIR}/bin && ln -snf ../openresty/luajit/bin/luajit ./luajit - COMMAND mkdir -p ${STAGE_EMBEDDED_DIR}/sbin && cd ${STAGE_EMBEDDED_DIR}/sbin && ln -snf ../openresty/nginx/sbin/nginx ./nginx -) diff --git a/build/cmake/package.cmake b/build/cmake/package.cmake deleted file mode 100644 index a70129f0a..000000000 --- a/build/cmake/package.cmake +++ /dev/null @@ -1,45 +0,0 @@ -add_custom_command( - OUTPUT ${STAMP_DIR}/package-bundle - DEPENDS - bundler - ${CMAKE_SOURCE_DIR}/build/package/Gemfile - ${CMAKE_SOURCE_DIR}/build/package/Gemfile.lock - COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} BUNDLE_GEMFILE=${CMAKE_SOURCE_DIR}/build/package/Gemfile BUNDLE_APP_CONFIG=${WORK_DIR}/src/package/.bundle bundle install --clean --path=${WORK_DIR}/src/package/bundle - COMMAND touch ${STAMP_DIR}/package-bundle -) - -add_custom_target( - package-core - DEPENDS ${STAMP_DIR}/package-bundle - COMMAND rm -rf ${WORK_DIR}/package-dest-core - COMMAND make - COMMAND make install-core DESTDIR=${WORK_DIR}/package-dest-core - COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} BUNDLE_GEMFILE=${CMAKE_SOURCE_DIR}/build/package/Gemfile BUNDLE_APP_CONFIG=${WORK_DIR}/src/package/.bundle WORK_DIR=${WORK_DIR} PACKAGE_WORK_DIR=${PACKAGE_WORK_DIR} PACKAGE=core ${CMAKE_SOURCE_DIR}/build/package/build_package - COMMAND rm -rf ${WORK_DIR}/package-dest-core -) - -if(ENABLE_HADOOP_ANALYTICS) - add_custom_target( - package-hadoop-analytics - DEPENDS ${STAMP_DIR}/package-bundle - COMMAND rm -rf ${WORK_DIR}/package-dest-hadoop-analytics - COMMAND make - COMMAND make install-hadoop-analytics DESTDIR=${WORK_DIR}/package-dest-hadoop-analytics - COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} BUNDLE_GEMFILE=${CMAKE_SOURCE_DIR}/build/package/Gemfile BUNDLE_APP_CONFIG=${WORK_DIR}/src/package/.bundle WORK_DIR=${WORK_DIR} PACKAGE_WORK_DIR=${PACKAGE_WORK_DIR} PACKAGE=hadoop-analytics ${CMAKE_SOURCE_DIR}/build/package/build_package - COMMAND rm -rf ${WORK_DIR}/package-dest-hadoop-analytics - ) -endif() - -# CMake policy CMP0037 to allow target named "package". -cmake_policy(PUSH) -if(POLICY CMP0037) - cmake_policy(SET CMP0037 OLD) -endif() -add_custom_target( - package - DEPENDS package-core -) -if(ENABLE_HADOOP_ANALYTICS) - add_dependencies(package package-hadoop-analytics) -endif() -cmake_policy(POP) diff --git a/build/cmake/perp.cmake b/build/cmake/perp.cmake deleted file mode 100644 index 974ede026..000000000 --- a/build/cmake/perp.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# Perp: Process supervision and control -ExternalProject_Add( - perp - URL http://b0llix.net/perp/distfiles/perp-${PERP_VERSION}.tar.gz - URL_HASH MD5=${PERP_HASH} - PATCH_COMMAND sed -i -e "s%BINDIR.*%BINDIR = ${INSTALL_PREFIX_EMBEDDED}/bin%" conf.mk - COMMAND sed -i -e "s%SBINDIR.*%SBINDIR = ${INSTALL_PREFIX_EMBEDDED}/sbin%" conf.mk - COMMAND sed -i -e "s%MANDIR.*%MANDIR = ${INSTALL_PREFIX_EMBEDDED}/share/man%" conf.mk - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND make - COMMAND make strip - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} -) diff --git a/build/cmake/rsyslog.cmake b/build/cmake/rsyslog.cmake deleted file mode 100644 index b99845cbf..000000000 --- a/build/cmake/rsyslog.cmake +++ /dev/null @@ -1,105 +0,0 @@ -# rsyslog: Log buffering and processing - -find_package(CURL REQUIRED) -require_program(autoconf) -require_program(automake) -require_program(libtool) -pkg_check_modules(LIBUUID REQUIRED uuid) - -# Build libestr dependency for rsyslog, since Ubuntu 12.04's package is too old -# and CentOS 6's package has some pkg-config issues, so it's not picked up -# (https://bugzilla.redhat.com/show_bug.cgi?id=1152899). -ExternalProject_Add( - libestr - URL http://libestr.adiscon.com/files/download/libestr-${LIBESTR_VERSION}.tar.gz - URL_HASH SHA256=${LIBESTR_HASH} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND /configure --prefix=${INSTALL_PREFIX_EMBEDDED} - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} -) - -ExternalProject_Add( - libfastjson - URL https://github.com/rsyslog/libfastjson/archive/v${LIBFASTJSON_VERSION}.tar.gz - URL_HASH MD5=${LIBFASTJSON_HASH} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND sh autogen.sh - COMMAND /configure --prefix=${INSTALL_PREFIX_EMBEDDED} - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} -) - -list(APPEND LIBLOGGING_CONFIGURE_CMD env) -list(APPEND LIBLOGGING_CONFIGURE_CMD /configure) -list(APPEND LIBLOGGING_CONFIGURE_CMD --prefix=${INSTALL_PREFIX_EMBEDDED}) -list(APPEND LIBLOGGING_CONFIGURE_CMD --disable-man-pages) -ExternalProject_Add( - liblogging - URL http://download.rsyslog.com/liblogging/liblogging-${LIBLOGGING_VERSION}.tar.gz - URL_HASH SHA256=${LIBLOGGING_HASH} - BUILD_IN_SOURCE 1 - # Run autoreconf to fix issues with the bundled configure file being built - # with specific versions of autoreconf and libtool that might be newer than - # the default OS packages. - CONFIGURE_COMMAND autoreconf --force --install -v - COMMAND ${LIBLOGGING_CONFIGURE_CMD} - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} - COMMAND chrpath -d ${STAGE_EMBEDDED_DIR}/bin/stdlogctl -) - -if(ENABLE_HADOOP_ANALYTICS) - # There's a small dependency on Python for librdkafka's Makefile: - # https://github.com/edenhill/librdkafka/blob/v0.9.2/Makefile#L8 - find_package(PythonInterp REQUIRED) - - ExternalProject_Add( - librdkafka - URL https://github.com/edenhill/librdkafka/archive/v${LIBRDKAFKA_VERSION}.tar.gz - URL_HASH MD5=${LIBRDKAFKA_HASH} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND /configure --prefix=${INSTALL_PREFIX_EMBEDDED} - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} - ) -endif() - -list(APPEND RSYSLOG_DEPENDS libestr) -list(APPEND RSYSLOG_DEPENDS libfastjson) -list(APPEND RSYSLOG_DEPENDS liblogging) -if(ENABLE_HADOOP_ANALYTICS) - list(APPEND RSYSLOG_DEPENDS librdkafka) -endif() - -list(APPEND RSYSLOG_CONFIGURE_CMD env) -list(APPEND RSYSLOG_CONFIGURE_CMD LIBESTR_CFLAGS=-I${STAGE_EMBEDDED_DIR}/include) -list(APPEND RSYSLOG_CONFIGURE_CMD "LIBESTR_LIBS=-L${STAGE_EMBEDDED_DIR}/lib -lestr") -list(APPEND RSYSLOG_CONFIGURE_CMD LIBFASTJSON_CFLAGS=-I${STAGE_EMBEDDED_DIR}/include/libfastjson) -list(APPEND RSYSLOG_CONFIGURE_CMD "LIBFASTJSON_LIBS=-L${STAGE_EMBEDDED_DIR}/lib -lfastjson") -list(APPEND RSYSLOG_CONFIGURE_CMD LIBLOGGING_STDLOG_CFLAGS=-I${STAGE_EMBEDDED_DIR}/include) -list(APPEND RSYSLOG_CONFIGURE_CMD "LIBLOGGING_STDLOG_LIBS=-L${STAGE_EMBEDDED_DIR}/lib -llogging-stdlog") -if(ENABLE_HADOOP_ANALYTICS) - list(APPEND RSYSLOG_CONFIGURE_CMD LIBRDKAFKA_CFLAGS=-I${STAGE_EMBEDDED_DIR}/include) - list(APPEND RSYSLOG_CONFIGURE_CMD "LIBRDKAFKA_LIBS=-L${STAGE_EMBEDDED_DIR}/lib -lrdkafka") -endif() -list(APPEND RSYSLOG_CONFIGURE_CMD /configure) -list(APPEND RSYSLOG_CONFIGURE_CMD --prefix=${INSTALL_PREFIX_EMBEDDED}) -list(APPEND RSYSLOG_CONFIGURE_CMD --enable-liblogging-stdlog) -list(APPEND RSYSLOG_CONFIGURE_CMD --disable-libgcrypt) -list(APPEND RSYSLOG_CONFIGURE_CMD --enable-imptcp) -list(APPEND RSYSLOG_CONFIGURE_CMD --enable-impstats) -list(APPEND RSYSLOG_CONFIGURE_CMD --enable-mmjsonparse) -list(APPEND RSYSLOG_CONFIGURE_CMD --enable-mmutf8fix) -list(APPEND RSYSLOG_CONFIGURE_CMD --enable-elasticsearch) -if(ENABLE_HADOOP_ANALYTICS) - list(APPEND RSYSLOG_CONFIGURE_CMD --enable-omkafka) -endif() - -ExternalProject_Add( - rsyslog - DEPENDS ${RSYSLOG_DEPENDS} - URL http://www.rsyslog.com/download/files/download/rsyslog/rsyslog-${RSYSLOG_VERSION}.tar.gz - URL_HASH SHA256=${RSYSLOG_HASH} - CONFIGURE_COMMAND rm -rf && mkdir -p # Clean across version upgrades - COMMAND ${RSYSLOG_CONFIGURE_CMD} - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} - COMMAND chrpath -d ${STAGE_EMBEDDED_DIR}/sbin/rsyslogd - COMMAND find ${STAGE_EMBEDDED_DIR}/lib/rsyslog/ -name *.so -exec chrpath -d {} $ -) diff --git a/build/cmake/ruby.cmake b/build/cmake/ruby.cmake deleted file mode 100644 index 1e9721034..000000000 --- a/build/cmake/ruby.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# Ruby & Bundler: For Rails web-app component -list(APPEND RUBY_CONFIGURE_CMD env) -list(APPEND RUBY_CONFIGURE_CMD /configure) -list(APPEND RUBY_CONFIGURE_CMD --prefix=${INSTALL_PREFIX_EMBEDDED}) -list(APPEND RUBY_CONFIGURE_CMD --enable-load-relative) -list(APPEND RUBY_CONFIGURE_CMD --disable-rpath) -list(APPEND RUBY_CONFIGURE_CMD --disable-install-doc) - -ExternalProject_Add( - ruby - URL https://cache.ruby-lang.org/pub/ruby/ruby-${RUBY_VERSION}.tar.bz2 - URL_HASH SHA256=${RUBY_HASH} - CONFIGURE_COMMAND rm -rf && mkdir -p # Clean across version upgrades - COMMAND ${RUBY_CONFIGURE_CMD} - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} -) - -ExternalProject_Add( - rubygems - DEPENDS ruby - URL https://rubygems.org/downloads/rubygems-update-${RUBYGEMS_VERSION}.gem - URL_HASH SHA256=${RUBYGEMS_HASH} - DOWNLOAD_NO_EXTRACT 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} gem update --system ${RUBYGEMS_VERSION} --no-document -) - -ExternalProject_Add( - bundler - DEPENDS rubygems - URL https://rubygems.org/downloads/bundler-${BUNDLER_VERSION}.gem - URL_HASH SHA256=${BUNDLER_HASH} - DOWNLOAD_NO_EXTRACT 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} gem install --no-document --env-shebang --local --force -) diff --git a/build/cmake/runit_svlogd.cmake b/build/cmake/runit_svlogd.cmake deleted file mode 100644 index 7ea4cd1b9..000000000 --- a/build/cmake/runit_svlogd.cmake +++ /dev/null @@ -1,10 +0,0 @@ -# runit's svlogd as alternative to perp's tinylog with more features. -ExternalProject_Add( - runit - URL http://smarden.org/runit/runit-${RUNIT_VERSION}.tar.gz - URL_HASH MD5=${RUNIT_HASH} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND cd runit-${RUNIT_VERSION}/src && make svlogd - INSTALL_COMMAND install -D -m 755 runit-${RUNIT_VERSION}/src/svlogd ${STAGE_EMBEDDED_DIR}/bin/svlogd -) diff --git a/build/cmake/static-site.cmake b/build/cmake/static-site.cmake deleted file mode 100644 index 69cb918a5..000000000 --- a/build/cmake/static-site.cmake +++ /dev/null @@ -1,14 +0,0 @@ -# api-umbrella-static-site: Example website content -ExternalProject_Add( - api_umbrella_static_site - DEPENDS bundler - URL https://github.com/NREL/api-umbrella-static-site/archive/${API_UMBRELLA_STATIC_SITE_VERSION}.tar.gz - URL_HASH MD5=${API_UMBRELLA_STATIC_SITE_HASH} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} bundle install --path=/vendor/bundle - BUILD_COMMAND env PATH=${DEV_INSTALL_PREFIX}/bin:${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} bundle exec middleman build - INSTALL_COMMAND rm -rf ${STAGE_EMBEDDED_DIR}/apps/static-site/releases - COMMAND mkdir -p ${STAGE_EMBEDDED_DIR}/apps/static-site/releases/0/build - COMMAND rsync -a /build/ ${STAGE_EMBEDDED_DIR}/apps/static-site/releases/0/build/ - COMMAND cd ${STAGE_EMBEDDED_DIR}/apps/static-site && ln -snf releases/0 ./current -) diff --git a/build/cmake/test/bundle.cmake b/build/cmake/test/bundle.cmake deleted file mode 100644 index 1869be32b..000000000 --- a/build/cmake/test/bundle.cmake +++ /dev/null @@ -1,33 +0,0 @@ -add_custom_command( - OUTPUT - ${STAMP_DIR}/test-bundle - ${WORK_DIR}/.bundle - ${WORK_DIR}/bundle - DEPENDS - bundler - ${CMAKE_SOURCE_DIR}/Gemfile - ${CMAKE_SOURCE_DIR}/Gemfile.lock - COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} BUNDLE_GEMFILE=${CMAKE_SOURCE_DIR}/Gemfile BUNDLE_APP_CONFIG=${WORK_DIR}/.bundle bundle install --clean --path=${WORK_DIR}/bundle - COMMAND touch -c ${WORK_DIR}/.bundle - COMMAND touch -c ${WORK_DIR}/bundle - COMMAND touch ${STAMP_DIR}/test-bundle -) - -# Normally we perform the bundle out-of-source (so the build takes place -# entirely out of source), but if testing/development is enabled for this -# build, then also create a local ".bundle/config" item within the source. This -# then allows for gems to be found when interacting with the local source -# version of the app. -if(ENABLE_TEST_DEPENDENCIES) - add_custom_command( - OUTPUT ${CMAKE_SOURCE_DIR}/.bundle/config - DEPENDS - ${STAMP_DIR}/test-bundle - ${WORK_DIR}/.bundle - ${WORK_DIR}/bundle - COMMAND rm -rf ${CMAKE_SOURCE_DIR}/.bundle - COMMAND ln -snf ${WORK_DIR}/.bundle ${CMAKE_SOURCE_DIR}/.bundle - COMMAND touch -c ${CMAKE_SOURCE_DIR}/.bundle/config - ) - add_custom_target(test-local-bundle ALL DEPENDS ${CMAKE_SOURCE_DIR}/.bundle/config) -endif() diff --git a/build/cmake/test/lua-deps.cmake b/build/cmake/test/lua-deps.cmake deleted file mode 100644 index 0f49e1087..000000000 --- a/build/cmake/test/lua-deps.cmake +++ /dev/null @@ -1,7 +0,0 @@ -# Luarocks test dependencies -test_luarocks_install(luacheck ${LUAROCK_LUACHECK_VERSION} ${LUAROCK_LUACHECK_HASH}) - -add_custom_target( - test-lua-deps - DEPENDS luarock_luacheck -) diff --git a/build/cmake/test/mailhog.cmake b/build/cmake/test/mailhog.cmake deleted file mode 100644 index 56f0c7bf7..000000000 --- a/build/cmake/test/mailhog.cmake +++ /dev/null @@ -1,10 +0,0 @@ -# MailHog: SMTP testing server -ExternalProject_Add( - mailhog - URL https://github.com/mailhog/MailHog/releases/download/v${MAILHOG_VERSION}/MailHog_linux_amd64 - URL_HASH MD5=${MAILHOG_HASH} - DOWNLOAD_NO_EXTRACT 1 - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND install -D -m 755 ${TEST_INSTALL_PREFIX}/bin/mailhog -) diff --git a/build/cmake/test/mongo-orchestration.cmake b/build/cmake/test/mongo-orchestration.cmake deleted file mode 100644 index 4e296bb79..000000000 --- a/build/cmake/test/mongo-orchestration.cmake +++ /dev/null @@ -1,11 +0,0 @@ -# Python test dependencies (mongo-orchestration) -add_custom_command( - OUTPUT ${TEST_INSTALL_PREFIX}/bin/pip - COMMAND virtualenv ${TEST_INSTALL_PREFIX} -) -add_custom_target(test_virtualenv ALL DEPENDS ${TEST_INSTALL_PREFIX}/bin/pip) -add_custom_command( - OUTPUT ${TEST_INSTALL_PREFIX}/bin/mongo-orchestration - COMMAND ${TEST_INSTALL_PREFIX}/bin/pip install --ignore-installed 'mongo-orchestration==${MONGO_ORCHESTRATION_VERSION}' -) -add_custom_target(test_pip_install ALL DEPENDS ${TEST_INSTALL_PREFIX}/bin/mongo-orchestration) diff --git a/build/cmake/test/openldap.cmake b/build/cmake/test/openldap.cmake deleted file mode 100644 index 347d906fc..000000000 --- a/build/cmake/test/openldap.cmake +++ /dev/null @@ -1,8 +0,0 @@ -# OpenLDAP: For testing LDAP admin auth. -ExternalProject_Add( - openldap - URL ftp://ftp.openldap.org/pub/OpenLDAP/openldap-release/openldap-${OPENLDAP_VERSION}.tgz - URL_HASH SHA1=${OPENLDAP_HASH} - CONFIGURE_COMMAND /configure --prefix=${TEST_INSTALL_PREFIX} --disable-backends --enable-mdb - BUILD_COMMAND make depend && make -) diff --git a/build/cmake/test/phantomjs.cmake b/build/cmake/test/phantomjs.cmake deleted file mode 100644 index 6d3a976e3..000000000 --- a/build/cmake/test/phantomjs.cmake +++ /dev/null @@ -1,9 +0,0 @@ -# PhantomJS: Headless WebKit for testing. -ExternalProject_Add( - phantomjs - URL https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-${PHANTOMJS_VERSION}-linux-x86_64.tar.bz2 - URL_HASH MD5=${PHANTOMJS_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND install -D -m 755 /bin/phantomjs ${TEST_INSTALL_PREFIX}/bin/phantomjs -) diff --git a/build/cmake/test/shellcheck.cmake b/build/cmake/test/shellcheck.cmake deleted file mode 100644 index c1817af14..000000000 --- a/build/cmake/test/shellcheck.cmake +++ /dev/null @@ -1,8 +0,0 @@ -ExternalProject_Add( - shellcheck - URL https://storage.googleapis.com/shellcheck/shellcheck-v${SHELLCHECK_VERSION}.linux.x86_64.tar.xz - URL_HASH SHA512=${SHELLCHECK_HASH} - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND install -D -m 755 /shellcheck ${TEST_INSTALL_PREFIX}/bin/shellcheck -) diff --git a/build/cmake/test/test.cmake b/build/cmake/test/test.cmake deleted file mode 100644 index 143349ea0..000000000 --- a/build/cmake/test/test.cmake +++ /dev/null @@ -1,21 +0,0 @@ -add_custom_target( - test-deps - DEPENDS ${STAMP_DIR}/test-bundle test-lua-deps -) - -add_custom_target( - test-target - DEPENDS test-deps - COMMAND env PATH=${STAGE_EMBEDDED_DIR}/bin:$ENV{PATH} BUNDLE_GEMFILE=${CMAKE_SOURCE_DIR}/Gemfile BUNDLE_APP_CONFIG=${WORK_DIR}/.bundle bundle exec rake -) - -# CMake policy CMP0037 business to allow target named "test". -cmake_policy(PUSH) -if(POLICY CMP0037) - cmake_policy(SET CMP0037 OLD) -endif() -add_custom_target( - test - DEPENDS all test-target -) -cmake_policy(POP) diff --git a/build/cmake/test/unbound.cmake b/build/cmake/test/unbound.cmake deleted file mode 100644 index 63fa9852c..000000000 --- a/build/cmake/test/unbound.cmake +++ /dev/null @@ -1,8 +0,0 @@ -# Unbound: Local DNS server for testing DNS changes -ExternalProject_Add( - unbound - URL http://www.unbound.net/downloads/unbound-${UNBOUND_VERSION}.tar.gz - URL_HASH SHA256=${UNBOUND_HASH} - BUILD_IN_SOURCE 1 - CONFIGURE_COMMAND /configure --prefix=${TEST_INSTALL_PREFIX} -) diff --git a/build/cmake/trafficserver.cmake b/build/cmake/trafficserver.cmake deleted file mode 100644 index 5566888a1..000000000 --- a/build/cmake/trafficserver.cmake +++ /dev/null @@ -1,22 +0,0 @@ -# TrafficServer: HTTP caching server -list(APPEND TRAFFICSERVER_CONFIGURE_CMD env) -list(APPEND TRAFFICSERVER_CONFIGURE_CMD SPHINXBUILD=false) -list(APPEND TRAFFICSERVER_CONFIGURE_CMD /configure) -list(APPEND TRAFFICSERVER_CONFIGURE_CMD --prefix=${INSTALL_PREFIX_EMBEDDED}) -list(APPEND TRAFFICSERVER_CONFIGURE_CMD --enable-experimental-plugins) - -ExternalProject_Add( - trafficserver - URL http://mirror.olnevhost.net/pub/apache/trafficserver/trafficserver-${TRAFFICSERVER_VERSION}.tar.bz2 - URL_HASH SHA512=${TRAFFICSERVER_HASH} - CONFIGURE_COMMAND rm -rf && mkdir -p # Clean across version upgrades - COMMAND ${TRAFFICSERVER_CONFIGURE_CMD} - INSTALL_COMMAND make install DESTDIR=${STAGE_DIR} - COMMAND chrpath -d ${STAGE_EMBEDDED_DIR}/lib/libtsmgmt.so - COMMAND find ${STAGE_EMBEDDED_DIR}/libexec/trafficserver/ -name *.so -exec chrpath -d {} $ - COMMAND find ${STAGE_EMBEDDED_DIR}/bin/ -name traffic_* -exec chrpath -d {} $ - COMMAND chrpath -d ${STAGE_EMBEDDED_DIR}/bin/traffic_top - # Trim our own distribution by removing some larger files we don't need for - # API Umbrella. - COMMAND rm -f ${STAGE_EMBEDDED_DIR}/bin/traffic_sac -) diff --git a/build/cmake/versions.cmake b/build/cmake/versions.cmake deleted file mode 100644 index eb1a672c1..000000000 --- a/build/cmake/versions.cmake +++ /dev/null @@ -1,108 +0,0 @@ -# Define the versions of the various dependencies to build. -set(API_UMBRELLA_STATIC_SITE_VERSION c02b8869cafb063deb7f9436d0137b0ea6e652aa) -set(API_UMBRELLA_STATIC_SITE_HASH 07dbd5e6d96e62a9ad6b725b14f727a1) -set(BUNDLER_VERSION 1.16.1) -set(BUNDLER_HASH 42b8e0f57093e1d10c15542f956a871446b759e7969d99f91caf3b6731c156e8) -set(ELASTICSEARCH_VERSION 2.4.6) -set(ELASTICSEARCH_HASH c3441bef89cd91206edf3cf3bd5c4b62550e60a9) -set(FLUME_VERSION 1.7.0) -set(FLUME_HASH 12496e632a96d7ca823ab3c239a2a7d2) -set(GOLANG_VERSION 1.8.3) -set(GOLANG_HASH 1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772) -set(KYLIN_VERSION 1.6.0) -set(KYLIN_HASH 3f15f35c5ad7168ab401858dd8759dee) -set(LIBCIDR_VERSION 1.2.3) -set(LIBCIDR_HASH c5efcc7ae114fdaa5583f58dacecd9de) -set(LIBESTR_VERSION 0.1.10) -set(LIBESTR_HASH bd655e126e750edd18544b88eb1568d200a424a0c23f665eb14bbece07ac703c) -set(LIBFASTJSON_VERSION 0.99.8) -set(LIBFASTJSON_HASH 730713ad1d851def7ac8898f751bbfdd) -set(LIBGEOIP_VERSION 1.6.12) -set(LIBGEOIP_HASH 77d496cc40daa1dbc2b97365807d64d7) -set(LIBLOGGING_VERSION 1.0.6) -set(LIBLOGGING_HASH 338c6174e5c8652eaa34f956be3451f7491a4416ab489aef63151f802b00bf93) -set(LIBRDKAFKA_VERSION 0.9.5) -set(LIBRDKAFKA_HASH 45bc9713bd4ed948e1efbd62688fc502) -set(LUAROCKS_VERSION 2.4.4) -set(LUAROCKS_HASH 04e8b19d565e86b1d08f745adc4b1a56) -set(LUAROCK_ARGPARSE_VERSION 0.5.0-1) -set(LUAROCK_ARGPARSE_HASH 02db647a5521809390b101a741cca4ff) -set(LUAROCK_CMSGPACK_VERSION 0.4.0-0) -set(LUAROCK_CMSGPACK_HASH f459d16fffdbbc85e582803321b3cec9) -set(LUAROCK_ICONV_VERSION 7-3) -set(LUAROCK_ICONV_HASH 138d21a895d267f09ff40fcb75324f74) -set(LUAROCK_INSPECT_VERSION 3.1.1-0) -set(LUAROCK_INSPECT_HASH 8a8a05f10b07a603e44e4f8b39bddd35) -set(LUAROCK_LIBCIDR_VERSION 0.1.2-1) -set(LUAROCK_LIBCIDR_HASH b6bdc9431cb488de8b58a83117107f7a) -set(LUAROCK_LUACHECK_VERSION 0.21.2-1) -set(LUAROCK_LUACHECK_HASH 2db2625f0c0008cfa1910b2b75926231) -set(LUAROCK_LUAPOSIX_VERSION 34.0.4-1) -set(LUAROCK_LUAPOSIX_HASH e584252902055ee40f250a1a304ec18e) -set(LUAROCK_LUATZ_VERSION 0.4-1) -set(LUAROCK_LUATZ_HASH 80772a925cea2ab53bc33184a6e3b24e) -set(LUAROCK_LUSTACHE_VERSION 1.3.1-0) -set(LUAROCK_LUSTACHE_HASH 840ecd41bf19ed1751916de2cd46229e) -set(LUAROCK_LYAML_VERSION 6.2.2-1) -set(LUAROCK_LYAML_HASH d8c8c11db09bfc3f82838d0195d7cf04) -set(LUAROCK_PENLIGHT_VERSION 1.5.4-1) -set(LUAROCK_PENLIGHT_HASH 8f4e6b4c7e851c28cb3e95be728d6507) -set(LUAROCK_RESTY_AUTO_SSL_VERSION 0.11.0-1) -set(LUAROCK_RESTY_AUTO_SSL_HASH f092ba7ff82c53e21e473222aadcf33e) -set(LUAROCK_RESTY_HTTP_VERSION 0.12-0) -set(LUAROCK_RESTY_HTTP_HASH deaf54d8ce752db7cae5a58566e11517) -set(LUAROCK_RESTY_UUID_VERSION 1.1-1) -set(LUAROCK_RESTY_UUID_HASH d14ae99d6f18edd5c934e6050e974c5e) -set(LUA_LUASOCKET_VERSION 652959890943c34d7180cae372339b91e62f0d7b) -set(LUA_LUASOCKET_HASH 6b3e3bdf60267f5957c2ea44e563ed70) -set(LUA_RESTY_DNS_CACHE_VERSION 32d9d461465edbec1cc798c18447c0ac7ee6e528) -set(LUA_RESTY_DNS_CACHE_HASH 3a5414110c6ad4331fe82873e19bd1e8) -set(LUA_RESTY_LOGGER_SOCKET_VERSION 15cc1c256e55b8e68ec9b220b6883c227a763d4e) -set(LUA_RESTY_LOGGER_SOCKET_HASH efe14697a8c4be612c011f54fce06191) -set(LUA_RESTY_SHCACHE_VERSION fb2e275c2cdca08eaa34a7b73375e41ac3eff200) -set(LUA_RESTY_SHCACHE_HASH 5d3cbcf8fbad1954cdcb3826afa41afe) -set(MAVEN_VERSION 3.5.0) -set(MAVEN_HASH 35c39251d2af99b6624d40d801f6ff02) -set(MAILHOG_VERSION 1.0.0) -set(MAILHOG_HASH 3b758c81bfe2c9110911511daca1a7bc) -set(MONGO_ORCHESTRATION_VERSION 0.6.10) -set(MONGODB_VERSION 3.2.19) -set(MONGODB_HASH 91ed5ed3b9531f664f3e8549026f0c1e) -set(MORA_VERSION 8127901857cf88d3f0902708b25ad930354973a3) -set(MORA_HASH b86cea913596370cd58fce89b23acd97) -set(NGX_DYUPS_VERSION a5e75737e04ff3e5040a80f5f739171e96c3359c) -set(NGX_DYUPS_HASH e16860efcd0629f38f514469052d998a) -set(NGX_TXID_VERSION f1c197cb9c42e364a87fbb28d5508e486592ca42) -set(NGX_TXID_HASH 408ee86eb6e42e27a51514f711c41d6b) -set(NODEJS_VERSION 8.11.1) -set(NODEJS_HASH 6617e245fa0f7fbe0e373e71d543fea878315324ab31dc64b4eba10e42d04c11) -set(OPENLDAP_VERSION 2.4.46) -set(OPENLDAP_HASH a9ae2273eb9bdd70090dafe0d018a3132606bef6) -set(OPENRESTY_VERSION 1.13.6.1) -set(OPENRESTY_HASH 637f82d0b36c74aec1c01bd3b8e0289c) -set(OPENSSL_VERSION 1.0.2o) -set(OPENSSL_HASH ec3f5c9714ba0fd45cb4e087301eb1336c317e0d20b575a125050470e8089e4d) -set(PCRE_VERSION 8.42) -set(PCRE_HASH 085b6aa253e0f91cae70b3cdbe8c1ac2) -set(PERP_VERSION 2.07) -set(PERP_HASH a2acc7425d556d9635a25addcee9edb5) -set(PHANTOMJS_VERSION 2.1.1) -set(PHANTOMJS_HASH 1c947d57fce2f21ce0b43fe2ed7cd361) -set(PRESTO_VERSION 0.173) -set(PRESTO_HASH f6586ac7a5001a771342307d44b1255d) -set(RUBY_VERSION 2.4.4) -set(RUBY_HASH 45a8de577471b90dc4838c5ef26aeb253a56002896189055a44dc680644243f1) -set(RUBYGEMS_VERSION 2.7.6) -set(RUBYGEMS_HASH ee5ef219ac97f5499c31e6071eae424c3265620ece33b5cc66e09fa30f22086a) -set(RSYSLOG_VERSION 8.33.0) -set(RSYSLOG_HASH 2df39d91baddb75c575aa03525cd9d4d20aad75011c6d6d25ef773ac26ff5c12) -set(RUNIT_VERSION 2.1.2) -set(RUNIT_HASH 6c985fbfe3a34608eb3c53dc719172c4) -set(SHELLCHECK_VERSION 0.4.7) -set(SHELLCHECK_HASH 64bf19a1292f0357c007b615150b6e58dba138bc7bf168c5a5e27016f8b4f802afd9950be8be46bf9e4833f98ae81c6e7b1761a3a76ddbba2a04929265433134) -set(TRAFFICSERVER_VERSION 7.1.6) -set(TRAFFICSERVER_HASH e08230cbe68f7e1647134a3f787b3e3867303d6be5fe144aaa13eb572863323b933836fdd2d9739b407fa7aa15066c251528c53379969eadff2322e72c4380f9) -set(UNBOUND_VERSION 1.7.0) -set(UNBOUND_HASH 94dd9071fb13d8ccd122a3ac67c4524a3324d0e771fc7a8a7c49af8abfb926a2) -set(YARN_VERSION 1.5.1) -set(YARN_HASH 561ac9089c33402abece941bc424cdd4) diff --git a/build/package/Gemfile.lock b/build/package/Gemfile.lock index cb1a88091..febcad6bf 100644 --- a/build/package/Gemfile.lock +++ b/build/package/Gemfile.lock @@ -10,7 +10,7 @@ GEM ffi (~> 1.0, >= 1.0.11) clamp (1.0.1) dotenv (2.2.0) - ffi (1.9.17) + ffi (1.9.25) fpm (1.8.0) archive-tar-minitar arr-pm (~> 0.0.10) @@ -45,4 +45,4 @@ DEPENDENCIES fpm (~> 1.8.0) BUNDLED WITH - 1.14.3 + 1.16.4 diff --git a/build/package/Makefile b/build/package/Makefile index d8e965df3..707fa5570 100644 --- a/build/package/Makefile +++ b/build/package/Makefile @@ -12,9 +12,6 @@ LOG_DIR:=$(WORK_DIR)/log docker_centos7 \ docker_centos7_build \ docker_centos7_verify \ - docker_ubuntu1204 \ - docker_ubuntu1204_build \ - docker_ubuntu1204_verify \ docker_ubuntu1404 \ docker_ubuntu1404_build \ docker_ubuntu1404_verify \ @@ -24,12 +21,12 @@ LOG_DIR:=$(WORK_DIR)/log docker_ubuntu1804 \ docker_ubuntu1804_build \ docker_ubuntu1804_verify \ - docker_debian7 \ - docker_debian7_build \ - docker_debian7_verify \ docker_debian8 \ docker_debian8_build \ docker_debian8_verify \ + docker_debian9 \ + docker_debian9_build \ + docker_debian9_verify \ docker_all $(LOG_DIR): @@ -64,15 +61,6 @@ docker_centos7_build: docker_centos7_verify: DIST=centos-7 $(MAKE) docker_verify -docker_ubuntu1204: - DIST=ubuntu-12.04 $(MAKE) docker - -docker_ubuntu1204_build: - DIST=ubuntu-12.04 $(MAKE) docker_build - -docker_ubuntu1204_verify: - DIST=ubuntu-12.04 $(MAKE) docker_verify - docker_ubuntu1404: DIST=ubuntu-14.04 $(MAKE) docker @@ -100,15 +88,6 @@ docker_ubuntu1804_build: docker_ubuntu1804_verify: DIST=ubuntu-18.04 $(MAKE) docker_verify -docker_debian7: - DIST=debian-7 $(MAKE) docker - -docker_debian7_build: - DIST=debian-7 $(MAKE) docker_build - -docker_debian7_verify: - DIST=debian-7 $(MAKE) docker_verify - docker_debian8: DIST=debian-8 $(MAKE) docker @@ -118,29 +97,35 @@ docker_debian8_build: docker_debian8_verify: DIST=debian-8 $(MAKE) docker_verify +docker_debian9: + DIST=debian-9 $(MAKE) docker + +docker_debian9_build: + DIST=debian-9 $(MAKE) docker_build + +docker_debian9_verify: + DIST=debian-9 $(MAKE) docker_verify + docker_all: docker_centos6 \ docker_centos7 \ - docker_ubuntu1204 \ docker_ubuntu1404 \ docker_ubuntu1604 \ docker_ubuntu1804 \ - docker_debian7 \ - docker_debian8 + docker_debian8 \ + docker_debian9 docker_all_build: docker_centos6_build \ docker_centos7_build \ - docker_ubuntu1204_build \ docker_ubuntu1404_build \ docker_ubuntu1604_build \ docker_ubuntu1804_build \ - docker_debian7_build \ - docker_debian8_build + docker_debian8_build \ + docker_debian9_build docker_all_verify: docker_centos6_verify \ docker_centos7_verify \ - docker_ubuntu1204_verify \ docker_ubuntu1404_verify \ docker_ubuntu1604_verify \ docker_ubuntu1804_verify \ - docker_debian7_verify \ - docker_debian8_verify + docker_debian8_verify \ + docker_debian9_verify diff --git a/build/package/build_package b/build/package/build_package index 8ba673639..1d2054f44 100755 --- a/build/package/build_package +++ b/build/package/build_package @@ -41,7 +41,7 @@ if [ "$PACKAGE" == "core" ]; then fpm_args+=("-d" "$dep") done - fpm_args+=("-C" "$WORK_DIR/package-dest-core") + fpm_args+=("-C" "$WORK_DESTDIR") fpm_args+=("--name" "api-umbrella") fpm_args+=("--config-files" "etc/api-umbrella/api-umbrella.yml") fpm_args+=("--after-install" "$source_dir/build/package/scripts/after-install") @@ -49,14 +49,6 @@ if [ "$PACKAGE" == "core" ]; then fpm_args+=("--after-remove" "$source_dir/build/package/scripts/after-remove") fpm_args+=("--directories" "/opt/api-umbrella") fpm_args+=("--directories" "/etc/api-umbrella") -elif [ "$PACKAGE" == "hadoop-analytics" ]; then - for dep in "${hadoop_analytics_package_dependencies[@]}"; do - fpm_args+=("-d" "$dep") - done - - fpm_args+=("-C" "$WORK_DIR/package-dest-hadoop-analytics") - fpm_args+=("--name" "api-umbrella-hadoop-analytics") - fpm_args+=("--depends" "api-umbrella") fi mkdir -p "$PACKAGE_WORK_DIR/build/$PACKAGE" diff --git a/build/package/docker_script b/build/package/docker_script index e58fe0c35..a87fe6817 100755 --- a/build/package/docker_script +++ b/build/package/docker_script @@ -8,7 +8,7 @@ if [ -f /etc/redhat-release ]; then yum -y install git rsync elif [ -f /etc/debian_version ]; then apt-get update - apt-get -y install git rsync + DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install git rsync fi # Create a clean copy of the source directory. @@ -22,7 +22,7 @@ cd "$clean_source_dir" git clean -d -f -x # Install any system dependencies for building. -"$clean_source_dir/build/scripts/install_build_dependencies" +"$clean_source_dir/tasks/install-system-build-dependencies" # shellcheck disable=SC1091 if [ -f /etc/os-release ]; then diff --git a/build/package/files/etc/logrotate.d/api-umbrella b/build/package/files/etc/logrotate.d/api-umbrella index 823304864..6a535bd53 100644 --- a/build/package/files/etc/logrotate.d/api-umbrella +++ b/build/package/files/etc/logrotate.d/api-umbrella @@ -42,7 +42,7 @@ # Rotate any remaining logs that can't be reopened, so we have to use the # "copytruncate" strategy. -/opt/api-umbrella/var/log/elasticsearch/*.log /opt/api-umbrella/var/log/trafficserver/*.blog /opt/api-umbrella/var/log/trafficserver/*.log /opt/api-umbrella/var/log/trafficserver/*.out { +/opt/api-umbrella/var/log/trafficserver/*.blog /opt/api-umbrella/var/log/trafficserver/*.log /opt/api-umbrella/var/log/trafficserver/*.out { daily rotate 90 missingok diff --git a/build/package/publish b/build/package/publish index 5c76b9447..f035e7e5c 100755 --- a/build/package/publish +++ b/build/package/publish @@ -10,9 +10,8 @@ source "$source_dir/build/package/parse_version" packages=( "build/package/work/current/centos-6/core/api-umbrella-$version-$package_iteration.el6.x86_64.rpm" "build/package/work/current/centos-7/core/api-umbrella-$version-$package_iteration.el7.x86_64.rpm" - "build/package/work/current/debian-7/core/api-umbrella_$version-$package_iteration~wheezy_amd64.deb" "build/package/work/current/debian-8/core/api-umbrella_$version-$package_iteration~jessie_amd64.deb" - "build/package/work/current/ubuntu-12.04/core/api-umbrella_$version-$package_iteration~precise_amd64.deb" + "build/package/work/current/debian-9/core/api-umbrella_$version-$package_iteration~stretch_amd64.deb" "build/package/work/current/ubuntu-14.04/core/api-umbrella_$version-$package_iteration~trusty_amd64.deb" "build/package/work/current/ubuntu-16.04/core/api-umbrella_$version-$package_iteration~xenial_amd64.deb" "build/package/work/current/ubuntu-18.04/core/api-umbrella_$version-$package_iteration~bionic_amd64.deb" diff --git a/build/package/scripts/after-install b/build/package/scripts/after-install index 7f15526ea..f34b0378f 100755 --- a/build/package/scripts/after-install +++ b/build/package/scripts/after-install @@ -31,76 +31,93 @@ esac if [ "$configure" = "true" ]; then user=api-umbrella group=api-umbrella + auto_ssl_user=api-umbrella-auto-ssl + auto_ssl_group=api-umbrella-auto-ssl deploy_user=api-umbrella-deploy deploy_group=api-umbrella-deploy - prefix_dir=/opt/api-umbrella + destdir="${DESTDIR:-}" + prefix_dir="$destdir${PREFIX:-/opt/api-umbrella}" embedded_dir=$prefix_dir/embedded + if [ -n "$destdir" ]; then + mkdir -p "$destdir/usr/bin" + mkdir -p "$destdir/var/log" + fi + # Create the main user & group. - if ! getent group $group > /dev/null; then - groupadd -r $group + if ! getent group "$group" > /dev/null; then + groupadd -r "$group" + fi + if ! getent passwd "$user" > /dev/null; then + useradd -r -g "$group" -s /sbin/nologin \ + -d "$prefix_dir" -c "API Umbrella user" "$user" + fi + + # Create the auto-ssl user & group. + if ! getent group "$auto_ssl_group" > /dev/null; then + groupadd -r "$auto_ssl_group" fi - if ! getent passwd $user > /dev/null; then - useradd -r -g $group -s /sbin/nologin \ - -d $prefix_dir -c "API Umbrella user" $user + if ! getent passwd "$auto_ssl_user" > /dev/null; then + useradd -r -g "$auto_ssl_group" -s /bin/nologin \ + -d "$prefix_dir" -c "API Umbrella auto-ssl user" "$auto_ssl_user" fi # Create the deploy user & group. - if ! getent group $deploy_group > /dev/null; then - groupadd -r $deploy_group + if ! getent group "$deploy_group" > /dev/null; then + groupadd -r "$deploy_group" fi - if ! getent passwd $deploy_user > /dev/null; then - useradd -r -g $deploy_group -s /bin/bash \ - -d /home/$deploy_user -c "API Umbrella deployment user" $deploy_user + if ! getent passwd "$deploy_user" > /dev/null; then + useradd -r -g "$deploy_group" -s /bin/bash \ + -d "$destdir/home/$deploy_user" -c "API Umbrella deployment user" "$deploy_user" fi - # Add the deploy user to the app group, so the deploy user can read config - # files. - if ! groups $deploy_user | grep -q -E "\\s$group(\\s|$)"; then - usermod -a -G $group $deploy_user + # Add the auto-ssl and deploy users to the app group, so these users can read + # config files. + if ! groups "$auto_ssl_user" | grep -q -E "\\s$group(\\s|$)"; then + usermod -a -G "$group" "$auto_ssl_user" + fi + if ! groups "$deploy_user" | grep -q -E "\\s$group(\\s|$)"; then + usermod -a -G "$group" "$deploy_user" fi # Fix previously created deploy user that couldn't actually login. - if getent passwd $deploy_user | grep -q "/sbin/nologin"; then - usermod -d /home/$deploy_user -s /bin/bash $deploy_user + if getent passwd "$deploy_user" | grep -q "/sbin/nologin"; then + usermod -d "$destdir/home/$deploy_user" -s /bin/bash "$deploy_user" fi # Create an empty .ssh/authorized_keys file with proper permissions if it # doesn't already exist. - if [ ! -f /home/$deploy_user/.ssh/authorized_keys ]; then - mkdir -p /home/$deploy_user/.ssh - touch /home/$deploy_user/.ssh/authorized_keys - chown -R $deploy_user:$deploy_group /home/$deploy_user - chmod 700 /home/$deploy_user - chmod 700 /home/$deploy_user/.ssh - chmod 600 /home/$deploy_user/.ssh/authorized_keys + if [ ! -f "$destdir/home/$deploy_user/.ssh/authorized_keys" ]; then + mkdir -p "$destdir/home/$deploy_user/.ssh" + touch "$destdir/home/$deploy_user/.ssh/authorized_keys" + chown -R "$deploy_user:$deploy_group" "$destdir/home/$deploy_user" + chmod 700 "$destdir/home/$deploy_user" + chmod 700 "$destdir/home/$deploy_user/.ssh" + chmod 600 "$destdir/home/$deploy_user/.ssh/authorized_keys" fi # Set file permissions - chown -R $user:$group $prefix_dir/etc $prefix_dir/var - chown -R $deploy_user:$deploy_group $embedded_dir/apps - if [ -d $embedded_dir/kylin ]; then - chown $user:$group $embedded_dir/kylin/tomcat/temp \ - $embedded_dir/kylin/tomcat/webapps \ - $embedded_dir/kylin/tomcat/logs - fi + chown -R "$user:$group" "$prefix_dir/etc" "$prefix_dir/var" + chown -R "$deploy_user:$deploy_group" "$embedded_dir/apps" # Re-create symlinks that may have inadvertently been cleaned up by the API # Umbrella v0.8 and v0.9 after-remove scripts during upgrades (this should be # fixed by the v0.10 after-remove script, so at some point, we can probably # remove this logic). - if [ ! -f /usr/bin/api-umbrella ]; then - cd /usr/bin && ln -snf ../..$prefix_dir/bin/api-umbrella ./api-umbrella + if [ ! -f "$destdir/usr/bin/api-umbrella" ]; then + cd "$destdir/usr/bin" && ln -snf "../..$prefix_dir/bin/api-umbrella" ./api-umbrella fi - if [ ! -f /var/log/api-umbrella ]; then - cd /var/log && ln -snf ../..$prefix_dir/var/log ./api-umbrella + if [ ! -f "$destdir/var/log/api-umbrella" ]; then + cd "$destdir/var/log" && ln -snf "../..$prefix_dir/var/log" ./api-umbrella fi - # Install service, but don't activate. - if command -v chkconfig > /dev/null 2>&1; then - chkconfig --add api-umbrella - elif command -v update-rc.d > /dev/null 2>&1; then - update-rc.d api-umbrella defaults 85 15 > /dev/null + if [ -z "$destdir" ]; then + # Install service, but don't activate. + if command -v chkconfig > /dev/null 2>&1; then + chkconfig --add api-umbrella + elif command -v update-rc.d > /dev/null 2>&1; then + update-rc.d api-umbrella defaults 85 15 > /dev/null + fi fi fi diff --git a/build/package/verify/Rakefile b/build/package/verify/Rakefile index ce7564259..fc937c312 100644 --- a/build/package/verify/Rakefile +++ b/build/package/verify/Rakefile @@ -8,6 +8,7 @@ namespace :spec do targets = [] Dir.glob("./spec/*").each do |dir| next unless File.directory?(dir) + target = File.basename(dir) target = "_#{target}" if target == "default" targets << target diff --git a/build/package/verify/docker_script b/build/package/verify/docker_script index 159a6e5e9..632892d35 100755 --- a/build/package/verify/docker_script +++ b/build/package/verify/docker_script @@ -35,14 +35,11 @@ if [ -f /etc/redhat-release ]; then socat \ sudo elif [ -f /etc/debian_version ]; then - apt-get -y install \ + DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install \ net-tools \ ruby \ socat \ sudo - if [ "$DIST" == "ubuntu-12.04" ] || [ "$DIST" == "debian-7" ]; then - apt-get -y install rubygems - fi fi gem install bundler --no-rdoc --no-ri diff --git a/build/package/verify/download_previous_packages b/build/package/verify/download_previous_packages index 773ad98a7..a83c98d2d 100755 --- a/build/package/verify/download_previous_packages +++ b/build/package/verify/download_previous_packages @@ -25,10 +25,11 @@ previous_versions=( distros=( centos-6 centos-7 - ubuntu-12.04 ubuntu-14.04 - debian-7 + ubuntu-16.04 + ubuntu-18.04 debian-8 + debian-9 ) if [ -n "${DIST:-}" ]; then @@ -53,10 +54,6 @@ for dist in "${distros[@]}"; do filename="api-umbrella-${version}.el7.x86_64.rpm" url_prefix="$bintray_url_root/api-umbrella-el7" ;; - ubuntu-12.04) - filename="api-umbrella_${version}~precise_amd64.deb" - url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella" - ;; ubuntu-14.04) filename="api-umbrella_${version}~trusty_amd64.deb" url_prefix="$bintray_url_root/api-umbrella-ubuntu/pool/main/a/api-umbrella" @@ -77,14 +74,18 @@ for dist in "${distros[@]}"; do filename="" fi ;; - debian-7) - filename="api-umbrella_${version}~wheezy_amd64.deb" - url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella" - ;; debian-8) filename="api-umbrella_${version}~jessie_amd64.deb" url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella" ;; + debian-9) + filename="api-umbrella_${version}~stretch_amd64.deb" + url_prefix="$bintray_url_root/api-umbrella-debian/pool/main/a/api-umbrella" + if [ "$(compare_version "$version")" -lt "$(compare_version "0.15.0")" ]; then + # No Debian 9 packages until v0.15 + filename="" + fi + ;; *) echo "Unknown distribution: $dist" exit 1 diff --git a/build/package/verify/spec/localhost/service_spec.rb b/build/package/verify/spec/localhost/service_spec.rb index bf4ac6fc5..93a2f73fe 100644 --- a/build/package/verify/spec/localhost/service_spec.rb +++ b/build/package/verify/spec/localhost/service_spec.rb @@ -249,7 +249,6 @@ def install_package(version) "mongod", "mora", "nginx", - "nginx-reloader", "rsyslog", "trafficserver", "web-delayed-job", @@ -257,7 +256,7 @@ def install_package(version) ].each do |service| # Make sure all the expected processes are reported as running and aren't # flapping up and down. - expect(command_result.stdout).to match(%r{^\[\+ \+\+\+ \+\+\+\] +#{service} +uptime: \d+s/\d+s +pids: \d+/\d+$}) + expect(output).to match(%r{^\[\+ \+\+\+ \+\+\+\] +#{service} +uptime: \d+s/\d+s +pids: \d+/\d+$}) end end @@ -268,7 +267,6 @@ def install_package(version) "/opt/api-umbrella/var/log/geoip-auto-updater/current", "/opt/api-umbrella/var/log/mongod/current", "/opt/api-umbrella/var/log/mora/current", - "/opt/api-umbrella/var/log/nginx-reloader/current", "/opt/api-umbrella/var/log/nginx/current", "/opt/api-umbrella/var/log/perpd/current", "/opt/api-umbrella/var/log/rsyslog/current", @@ -367,7 +365,7 @@ def install_package(version) http.verify_mode = OpenSSL::SSL::VERIFY_NONE response = http.request(Net::HTTP::Get.new(uri.request_uri)) expect(response.code).to eql("403") - expect(response.body).to include("API_KEY_MISSING") + expect(response.body).to include("API_KEY_OR_TOKEN_MISSING") end it "gatekeeper blocks invalid key requests" do diff --git a/build/package_dependencies.sh b/build/package_dependencies.sh index 78e63097a..78233124f 100644 --- a/build/package_dependencies.sh +++ b/build/package_dependencies.sh @@ -8,12 +8,12 @@ if [ -f /etc/os-release ]; then fi if [ -f /etc/redhat-release ]; then - util_linux_package="util-linux-ng" - procps_package="procps" + util_linux_package="util-linux" + procps_package="procps-ng" - if [[ "${VERSION_ID:-}" == "7" ]]; then - util_linux_package="util-linux" - procps_package="procps-ng" + if [[ "${VERSION_ID:-}" == "6" ]]; then + util_linux_package="util-linux-ng" + procps_package="procps" fi core_package_dependencies=( @@ -57,9 +57,16 @@ if [ -f /etc/redhat-release ]; then # For OpenResty's "resty" CLI. perl perl-Time-HiRes - ) - hadoop_analytics_package_dependencies=( - java-1.8.0-openjdk-headless + + # lua-icu-date + libicu-devel + + # nokogiri + libxml2-devel + libxslt-devel + + # For prefixed console output (gnu version for strftime support). + gawk ) core_build_dependencies=( autoconf @@ -77,7 +84,6 @@ if [ -f /etc/redhat-release ]; then libyaml-devel make ncurses-devel - openssl openssl-devel patch pcre-devel @@ -89,9 +95,9 @@ if [ -f /etc/redhat-release ]; then tcl-devel unzip xz - ) - hadoop_analytics_build_dependencies=( - java-1.8.0-openjdk-devel + + # For "unbuffer" command for Taskfile. + expect ) test_build_dependencies=( # Binary and readelf tests @@ -117,25 +123,32 @@ if [ -f /etc/redhat-release ]; then # OpenLDAP groff + + # For running lsof tests in Docker as root + sudo ) elif [ -f /etc/debian_version ]; then - libffi_version=6 - openjdk_version=7 - - if [[ "$ID" == "debian" && "$VERSION_ID" == "7" ]]; then - libffi_version=5 - elif [[ "$ID" == "ubuntu" && "$VERSION_ID" == "16.04" ]]; then - openjdk_version=8 - elif [[ "$ID" == "ubuntu" && "$VERSION_ID" == "18.04" ]]; then - openjdk_version=8 - export DEBIAN_FRONTEND=noninteractive + libcurl_version=3 + libtool_bin_package="libtool-bin" + openjdk_version=8 + + if [[ "$ID" == "ubuntu" && "$VERSION_ID" == "18.04" ]]; then + libcurl_version=4 + fi + + if [[ "$ID" == "ubuntu" && "$VERSION_ID" == "14.04" ]]; then + libtool_bin_package="libtool" + fi + + if [[ "$ID" == "debian" && "$VERSION_ID" == "8" ]] || [[ "$ID" == "ubuntu" && "$VERSION_ID" == "14.04" ]]; then + openjdk_version=7 fi core_package_dependencies=( # General bash libc6 - "libffi$libffi_version" + libffi6 libncurses5 libpcre3 libuuid1 @@ -157,7 +170,7 @@ elif [ -f /etc/debian_version ]; then "openjdk-$openjdk_version-jre-headless" # rsyslog omelasticsearch - libcurl4 + "libcurl$libcurl_version" # init.d script helpers sysvinit-utils @@ -171,9 +184,16 @@ elif [ -f /etc/debian_version ]; then # For OpenResty's "resty" CLI. perl - ) - hadoop_analytics_package_dependencies=( - "openjdk-$openjdk_version-jre-headless" + + # lua-icu-date + libicu-dev + + # nokogiri + libxml2-dev + libxslt-dev + + # For prefixed console output (gnu version for strftime support). + gawk ) core_build_dependencies=( autoconf @@ -189,11 +209,11 @@ elif [ -f /etc/debian_version ]; then libpcre3-dev libssl-dev libtool + "$libtool_bin_package" libxml2-dev libyaml-dev lsb-release make - openssl patch pkg-config python @@ -203,10 +223,9 @@ elif [ -f /etc/debian_version ]; then unzip uuid-dev xz-utils - nodejs - ) - hadoop_analytics_build_dependencies=( - "openjdk-$openjdk_version-jdk" + + # For "unbuffer" command for Taskfile. + expect ) test_build_dependencies=( # Binary and readelf tests @@ -232,10 +251,13 @@ elif [ -f /etc/debian_version ]; then # OpenLDAP groff-base + + # For running lsof tests in Docker as root + sudo ) - if [[ "$ID" == "debian" && "$VERSION_ID" == "8" ]] || [[ "$ID" == "ubuntu" && "$VERSION_ID" == "16.04" ]] || [[ "$ID" == "ubuntu" && "$VERSION_ID" == "18.04" ]] ; then - core_build_dependencies+=("libtool-bin") + if [[ "$ID" != "ubuntu" || "$VERSION_ID" != "14.04" ]]; then + test_build_dependencies+=("virtualenv") fi else echo "Unknown build system" @@ -244,9 +266,7 @@ fi all_build_dependencies=( "${core_package_dependencies[@]}" - "${hadoop_analytics_package_dependencies[@]}" "${core_build_dependencies[@]}" - "${hadoop_analytics_build_dependencies[@]}" ) # shellcheck disable=SC2034 diff --git a/build/patches/openresty-cli.patch b/build/patches/openresty-cli.patch new file mode 100644 index 000000000..664aa6549 --- /dev/null +++ b/build/patches/openresty-cli.patch @@ -0,0 +1,43 @@ +diff --git a/configure b/configure +index 0a4f09b..e7d0d0c 100755 +--- a/configure ++++ b/configure +@@ -1096,38 +1096,6 @@ _EOC_ + } + push @make_install_cmds, "cd $root_dir/build/$resty_cli_dir && " + . "$root_dir/build/install bin/* $target_dir"; +- +- if ($platform ne 'msys') { +- # patch the resty script: +- +- print "patching the resty script with hard-coded nginx binary ", +- "path...\n"; +- +- my $resty_bin = "$root_dir/build/$resty_cli_dir/bin/resty"; +- open my $in, $resty_bin +- or die "Cannot open $resty_bin for reading: $!\n"; +- my ($new, $found); +- while (<$in>) { +- if (/^my \$nginx_path;$/) { +- $found = 1; +- $new .= qq/my \$nginx_path = '$ngx_sbin';\n/; +- +- } else { +- $new .= $_; +- } +- } +- close $in; +- +- if (!$found) { +- die "failed to patch $resty_bin: the line 'my \$nginx_path' ", +- "was not found.\n"; +- } +- +- open my $out, ">$resty_bin" +- or die "Cannot open $resty_bin for writing: $!\n"; +- print $out $new; +- close $out; +- } + } + + # configure restydoc indexes diff --git a/build/patches/openresty-opm.patch b/build/patches/openresty-opm.patch new file mode 100644 index 000000000..db3a77437 --- /dev/null +++ b/build/patches/openresty-opm.patch @@ -0,0 +1,15 @@ +diff --git a/bundle/opm-0.0.5/bin/opm b/bundle/opm-0.0.5/bin/opm +index 9029225..24a8200 100755 +--- a/bundle/opm-0.0.5/bin/opm ++++ b/bundle/opm-0.0.5/bin/opm +@@ -97,10 +97,6 @@ if ($cmd eq '-v') { + exit; + } + +-# explicitly clear the environments to avoid breaking luajit and resty. +-delete $ENV{LUA_PATH}; +-delete $ENV{LUA_CPATH}; +- + for ($cmd) { + if ($_ eq 'get' || $_ eq 'install') { + check_lock_file(); diff --git a/build/scripts/distclean b/build/scripts/distclean deleted file mode 100755 index 9accefe14..000000000 --- a/build/scripts/distclean +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -e -u -x - -source_dir="$(dirname "$(dirname "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)")")" - -if [ ! -f "$source_dir/CMakeLists.txt" ]; then - echo "Can't find root source directory" - exit 1 -fi - -rm -rf \ - "$source_dir/CMakeCache.txt" \ - "$source_dir/CMakeFiles"/* \ - "$source_dir/Makefile" \ - "$source_dir/build/package/.bundle" \ - "$source_dir/build/package/vendor" \ - "$source_dir/build/scripts/.bundle" \ - "$source_dir/build/scripts/vendor" \ - "$source_dir/build/work"/* \ - "$source_dir/cmake_install.cmake" \ - "$source_dir/install_manifest.txt" \ - "$source_dir/src/api-umbrella/hadoop-analytics"/*/dependency-reduced-pom.xml \ - "$source_dir/src/api-umbrella/web-app/.bundle" \ - "$source_dir/src/api-umbrella/web-app/brakeman.html" \ - "$source_dir/src/api-umbrella/web-app/log"/* \ - "$source_dir/src/api-umbrella/web-app/public/test-assets" \ - "$source_dir/src/api-umbrella/web-app/spec/reports" \ - "$source_dir/src/api-umbrella/web-app/tmp"/* \ - "$source_dir/test/config/.overrides.yml" \ - "$source_dir/test/tmp"/* diff --git a/build/scripts/download_cmake b/build/scripts/download_cmake deleted file mode 100755 index 984b8ff46..000000000 --- a/build/scripts/download_cmake +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash - -set -e -u - -version=3.8.2 -checksum=33e4851d3219b720f4b64fcf617151168f1bffdf5afad25eb4b7f5f58cee3a08 - -build_dir=$(pwd) -work_dir="$build_dir/build/work" -version_stamp="$work_dir/cmake/version-$version" -prefix="$work_dir/cmake" - -if [[ ! -e "$prefix/bin/cmake" || ! -e "$version_stamp" ]]; then - rm -rf "$prefix" - mkdir -p "$prefix" - - download_filename="cmake-$version-Linux-x86_64.tar.gz" - minor_version=${version%.*} - download_url="https://cmake.org/files/v$minor_version/$download_filename" - download_path="$work_dir/$download_filename" - if [ ! -e "$download_path" ]; then - echo "Downloading CMake $version..." - curl -fL -o "$download_path" "$download_url" - fi - - actual_checksum=$(openssl sha256 "$download_path" | awk '{print $2}') - if [ "$actual_checksum" != "$checksum" ]; then - echo "SHA256 hash of" - echo " $download_path" - echo "does not match expected value" - echo " expected: '$checksum'" - echo " actual: '$actual_checksum'" - exit 1 - fi - - echo "Extracting CMake $version..." - tar --strip-components 1 -C "$prefix" -xf "$download_path" - rm -f "$download_path" - touch "$version_stamp" - echo "Completed local CMake $version installation into $prefix" -fi diff --git a/circle.yml b/circle.yml deleted file mode 100644 index d5eff625d..000000000 --- a/circle.yml +++ /dev/null @@ -1,54 +0,0 @@ -general: - artifacts: - # Keep logs after running to help debug if errors do crop up. - - test/tmp/run/api-umbrella-root/var/log - - src/api-umbrella/web-app/log - - src/api-umbrella/web-app/brakeman.html - # Keep screenshots of capybara failures for easier debugging. - - test/tmp/capybara -machine: - pre: - # Enable IPv6 on CircleCI for running IPv6 integration tests. - - sudo sysctl -w net.ipv6.conf.lo.disable_ipv6=0 net.ipv6.conf.default.disable_ipv6=0 net.ipv6.conf.all.disable_ipv6=0 - # Disable RVM so the various environment variables (things like GEM_HOME) - # don't conflict with our embedded ruby installation. - - rvm reset -dependencies: - cache_directories: - - build/work - - CMakeCache.txt - - CMakeFiles - - cmake_install.cmake - pre: - # Install any system package dependencies. - - sudo ./build/scripts/install_build_dependencies - # Set the local file timestamps based on when they were last modified in - # git. This helps prevent unnecessary rebuilds in the CI environment, since - # make is dependent on file timestamps and every CI test run operates on a - # fresh clone. - - if [[ ! -e build/work/git-restore-mtime ]]; then mkdir -p build/work && curl -f -L https://raw.githubusercontent.com/MestreLion/git-tools/0fc841a3e49d041576e5b21d1644c8df2d2ef801/git-restore-mtime > build/work/git-restore-mtime && chmod +x build/work/git-restore-mtime; fi - # - ./build/work/git-restore-mtime -f . - override: - # Build all the API Umbrella software dependencies. - - ./configure --enable-test-dependencies - - make all test-deps - # Remove the download archives, since we don't need to cache these in - # CircleCI, and doing so also leads to multiple versions being kept around - # whenever we bump our dependency versions. - - make clean-download-archives -database: - override: - # Don't perform any database tasks that CircleCI infers, since they're not - # needed. - - /bin/true -compile: - override: - # Don't perform any compile tasks that CircleCI infers. We'll run our - # tasks. - - /bin/true -test: - override: - - ./test/scripts/circle-ci: - parallel: true - files: - - test/**/test_*.rb diff --git a/config/default.yml b/config/default.yml index 8c1a95976..97008c3a1 100644 --- a/config/default.yml +++ b/config/default.yml @@ -11,14 +11,21 @@ rlimits: nproc: 20000 http_port: 80 https_port: 443 +listen: + addresses: + - "*" + - "[::]" nginx: workers: auto worker_connections: 8192 + error_log_level: warn access_log_filename: access.log access_log_options: buffer=256k flush=10s proxy_connect_timeout: 60 proxy_read_timeout: 60 proxy_send_timeout: 60 + proxy_buffer_size: 8k + proxy_buffers: 8 8k keepalive_timeout: 75 ssl_protocols: "TLSv1 TLSv1.1 TLSv1.2" ssl_ciphers: "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS" @@ -30,29 +37,21 @@ nginx: ssl_ecdh_curve: secp384r1 lua_ssl_trusted_certificate: lua_ssl_verify_depth: 1 - dyups: - host: 127.0.0.1 - port: 14005 shared_dicts: active_config: size: 3m api_users: size: 3m - dns_cache: - size: 300k geocode_city_cache: size: 100k interval_locks: size: 20k locks: size: 20k - resolved_hosts: - size: 100k stats: size: 20m upstream_checksums: size: 200k - _reloader_frequency: 14400 # 4 hours gatekeeper: api_key_methods: - fiware-oauth2 @@ -67,14 +66,16 @@ gatekeeper: - github-oauth2 - facebook-oauth2 - google-oauth2 + - keycloak-oauth2 default_idp: backend_name: fiware-oauth2 host: http://idm.docker:3000 + realm: demo + jwt_enabled: false + key: pubkey trafficserver: host: 127.0.0.1 port: 14009 - autoconf_port: 14007 - management_port: 14008 storage: size: 256M embedded_server_config: @@ -92,6 +93,7 @@ web: min_threads: 2 max_threads: 24 api_user: + email_regex: "\\A[^@\\s]+@[^@\\s]+\\.[^@\\s]+\\z" first_name_exclude_regex: "(http|https|www|<|>|\\r|\\n)" last_name_exclude_regex: "(http|https|www|<|>|\\r|\\n)" admin: @@ -99,13 +101,17 @@ web: username_is_email: true password_length_min: 14 password_length_max: 72 - email_regex: "\\A[^@\\s]+@[^@\\s]+\\z" + email_regex: "\\A[^@\\s]+@[^@\\s]+\\.[^@\\s]+\\z" password_regex: auth_strategies: enabled: - local cas: - options: {} + options: + service_validate_url: /serviceValidate + login_url: /login + logout_url: /logout + ssl: true facebook: client_id: client_secret: @@ -119,9 +125,18 @@ web: client_id: client_secret: ldap: - options: {} + options: + port: 389 + method: plain + uid: sAMAccountName max.gov: require_mfa: true + options: + host: login.max.gov + login_url: /cas/login + service_validate_url: /cas/serviceValidate + logout_url: /cas/logout + ssl: true mailer: smtp_settings: static_site: @@ -129,9 +144,8 @@ static_site: port: 14013 router: api_backends: - host: 127.0.0.1 - port: 14011 - keepalive_connections: 10 + keepalive_connections: 20 + keepalive_idle_timeout: 120 trusted_proxies: [] global_rate_limits: ip_rate: @@ -142,18 +156,34 @@ router: ip_connections_size: 5m ip_connections_log_level: error web_app_host: "*" - web_app_backend_regex: "^/(admin|admins|web-assets)(/|$)" - web_app_backend_required_https_regex: "^.*" website_backend_required_https_regex_default: "^.*" redirect_not_found_to_https: true +auto_ssl: + workers: 1 + worker_connections: 8192 + http: + port: 14005 + https: + port: 14006 + user: api-umbrella-auto-ssl + group: api-umbrella-auto-ssl + hook_server: + port: 14007 rsyslog: host: 127.0.0.1 port: 14014 +log: + destination: file dns_resolver: negative_ttl: 60 max_stale: 86400 timeout: 2000 retries: 3 + # This default could be revisited, but historically we didn't resolve AAAA + # records for API backends, so for compatibility keep this disabled by + # default. Enabling may also break certain hosting environments that still + # aren't IPv6 compatible. + allow_ipv6: false mongodb: url: "mongodb://127.0.0.1:14001/api_umbrella" read_preference: primaryPreferred @@ -173,6 +203,7 @@ elasticsearch: embedded_server_env: heap_size: 512m api_version: 2 + template_version: 1 embedded_server_config: network: host: 127.0.0.1 @@ -190,42 +221,15 @@ elasticsearch: breaker: fielddata: limit: 60% - index: - translog: - # Sync the data to disk asynchronously on a fixed interval, rather than - # for every request. This significantly helps indexing throughput (at - # the risk of losing a few seconds of data if things crash). - durability: async - sync_interval: 10s + aws_signing_proxy: + host: 127.0.0.1 + port: 14017 + workers: 1 + error_log_level: notice analytics: adapter: elasticsearch timezone: UTC log_request_url_query_params_separately: false -kylin: - protocol: http - host: 127.0.0.1 - port: 7070 -presto: - protocol: http - host: 127.0.0.1 - port: 14015 - embedded_server_config: - http_port: 14015 - max_memory: 2GB - max_memory_per_node: 1GB - discovery_uri: http://127.0.0.1:14015 - query_max_run_time: 60s - hive: - metastore_uri: thrift://127.0.0.1:9083 -flume: - host: 127.0.0.1 - port: 14016 - embedded_server_config: - java_opts: -kafka: - brokers: [] - topic: api_umbrella_logs -log_template_version: v1 strip_cookies: - ^__utm.*$ - ^_ga$ @@ -334,7 +338,7 @@ internal_apis: - _id: api-umbrella-gatekeeper-backend name: API Umbrella - Gatekeeper APIs frontend_host: "{{router.web_app_host}}" - backend_host: 127.0.0.1 + backend_host: ~ backend_protocol: http balance_algorithm: least_conn sort_order: 1 @@ -346,6 +350,8 @@ internal_apis: backend_prefix: "/api-umbrella/v1/health" - frontend_prefix: "/api-umbrella/v1/state" backend_prefix: "/api-umbrella/v1/state" + settings: + require_https: required_return_error sub_settings: - http_method: get regex: "^/api-umbrella/v1/(health|state)" @@ -353,24 +359,58 @@ internal_apis: disable_api_key: true rate_limit_mode: unlimited require_https: optional - - _id: api-umbrella-web-backend - name: API Umbrella - Web APIs + disable_analytics: true + - _id: api-umbrella-web-app-backend + name: API Umbrella - HTTP APIs frontend_host: "{{router.web_app_host}}" - backend_host: 127.0.0.1 + backend_host: ~ backend_protocol: http balance_algorithm: least_conn - sort_order: 1 + sort_order: 2 servers: - host: "{{web.host}}" port: "{{web.port}}" url_matches: - frontend_prefix: "/api-umbrella/" backend_prefix: "/api-umbrella/" + - frontend_prefix: "/admin/" + backend_prefix: "/admin/" + - frontend_prefix: "/admin" + backend_prefix: "/admin" + exact_match: true + - frontend_prefix: "/admins/" + backend_prefix: "/admins/" + - frontend_prefix: "/admins" + backend_prefix: "/admins" + exact_match: true + - frontend_prefix: "/web-assets/" + backend_prefix: "/web-assets/" + settings: + require_https: required_return_error sub_settings: - - http_method: post - regex: "^/api-umbrella/v1/users" - - http_method: post - regex: "^/api-umbrella/v1/contact" + - http_method: any + regex: "^/admin/stats" + settings: + disable_api_key: true + - http_method: POST + regex: "^/admin/login" + settings: + disable_api_key: true + rate_limit_mode: custom + rate_limits: + - duration: 15000 + accuracy: 1000 + limit_by: ip + limit: 100 + distributed: true + response_headers: true + - http_method: any + regex: "^/(admin|web-assets)" + settings: + disable_api_key: true + rate_limit_mode: unlimited + redirect_https: true + disable_analytics: true internal_website_backends: - _id: api-umbrella-website-backend frontend_host: "{{router.web_app_host}}" @@ -387,3 +427,8 @@ ban: ember_server: port: 14050 live_reload_port: 14051 +umask: "0027" +_test_config: + default_null_override_hash: + default_null_override_string: + default_empty_hash_override_hash: {} diff --git a/config/elasticsearch_templates.json b/config/elasticsearch_templates_v1_es2.json similarity index 84% rename from config/elasticsearch_templates.json rename to config/elasticsearch_templates_v1_es2.json index cd758bb14..107412363 100644 --- a/config/elasticsearch_templates.json +++ b/config/elasticsearch_templates_v1_es2.json @@ -5,7 +5,8 @@ "template": "api-umbrella-logs-v1-*", "settings": { "index": { - "number_of_shards": 3 + "number_of_shards": 3, + "codec": "best_compression" }, "analysis": { "analyzer": { @@ -15,6 +16,10 @@ "filter": ["lowercase"] } } + }, + "translog": { + "durability": "async", + "sync_interval": "10s" } }, "mappings": { @@ -22,21 +27,7 @@ "_all": { "enabled": false }, - "date_detection": false, - "numeric_detection": false, - "dynamic_templates": [ - { - "string_template": { - "match": "*", - "match_mapping_type": "string", - "mapping": { - "type": "string", - "index": "analyzed", - "analyzer": "keyword_lowercase" - } - } - } - ], + "dynamic": false, "properties": { "api_backend_id": { "type": "string", @@ -50,22 +41,10 @@ "type": "string", "index": "not_analyzed" }, - "backend_response_time": { - "type": "integer" - }, "gatekeeper_denied_code": { "type": "string", "analyzer": "keyword_lowercase" }, - "internal_gatekeeper_time": { - "type": "float" - }, - "internal_response_time": { - "type": "float" - }, - "proxy_overhead": { - "type": "integer" - }, "request_accept": { "type": "string", "analyzer": "keyword_lowercase" @@ -109,10 +88,6 @@ "type": "string", "index": "not_analyzed" }, - "request_ip_location": { - "type": "geo_point", - "lat_lon": true - }, "request_ip_region": { "type": "string", "index": "not_analyzed" @@ -195,14 +170,14 @@ "type": "string", "analyzer": "keyword_lowercase" }, - "user_email": { - "type": "string", - "analyzer": "keyword_lowercase" - }, "user_id": { "type": "string", "index": "not_analyzed" }, + "user_email": { + "type": "string", + "analyzer": "keyword_lowercase" + }, "user_registration_source": { "type": "string", "analyzer": "keyword_lowercase" diff --git a/config/elasticsearch_templates_v1_es5.json b/config/elasticsearch_templates_v1_es5.json new file mode 100644 index 000000000..393f5e1d4 --- /dev/null +++ b/config/elasticsearch_templates_v1_es5.json @@ -0,0 +1,192 @@ +[ + { + "id": "api-umbrella-log-v1-template", + "template": { + "template": "api-umbrella-logs-v1-*", + "settings": { + "index": { + "number_of_shards": 3, + "codec": "best_compression" + }, + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "filter": ["lowercase"] + }, + "uppercase_normalizer": { + "type": "custom", + "filter": ["uppercase"] + } + } + }, + "translog": { + "durability": "async", + "sync_interval": "10s" + } + }, + "mappings": { + "log": { + "_all": { + "enabled": false + }, + "dynamic": false, + "properties": { + "api_backend_id": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "api_backend_url_match_id": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "api_key": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "gatekeeper_denied_code": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_accept": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_accept_encoding": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_at": { + "type": "date" + }, + "request_basic_auth_username": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_connection": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_content_type": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_hierarchy": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_host": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_ip": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_ip_city": { + "type": "keyword" + }, + "request_ip_country": { + "type": "keyword", + "normalizer": "uppercase_normalizer" + }, + "request_ip_region": { + "type": "keyword", + "normalizer": "uppercase_normalizer" + }, + "request_method": { + "type": "keyword", + "normalizer": "uppercase_normalizer" + }, + "request_origin": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_path": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_referer": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_scheme": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_size": { + "type": "integer" + }, + "request_url": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_url_query": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_user_agent": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_user_agent_family": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_user_agent_type": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_age": { + "type": "integer" + }, + "response_cache": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_content_encoding": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_content_length": { + "type": "integer" + }, + "response_content_type": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_server": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_size": { + "type": "integer" + }, + "response_status": { + "type": "short" + }, + "response_time": { + "type": "integer" + }, + "response_transfer_encoding": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "user_id": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "user_email": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "user_registration_source": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + } + } + } + } + } + } +] diff --git a/config/elasticsearch_templates_v2_es5.json b/config/elasticsearch_templates_v2_es5.json new file mode 100644 index 000000000..b270b9335 --- /dev/null +++ b/config/elasticsearch_templates_v2_es5.json @@ -0,0 +1,217 @@ +[ + { + "id": "api-umbrella-log-v2-template", + "template": { + "template": "api-umbrella-logs-v2-*", + "settings": { + "index": { + "number_of_shards": 1, + "codec": "best_compression" + }, + "analysis": { + "normalizer": { + "lowercase_normalizer": { + "type": "custom", + "filter": ["lowercase"] + }, + "uppercase_normalizer": { + "type": "custom", + "filter": ["uppercase"] + } + } + }, + "translog": { + "durability": "async", + "sync_interval": "10s" + } + }, + "mappings": { + "log": { + "_all": { + "enabled": false + }, + "dynamic": "strict", + "properties": { + "api_backend_id": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "api_backend_url_match_id": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "api_key": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "gatekeeper_denied_code": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_accept": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_accept_encoding": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_at": { + "type": "date" + }, + "request_basic_auth_username": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_connection": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_content_type": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_hierarchy": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_host": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_ip": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_ip_city": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_ip_country": { + "type": "keyword", + "normalizer": "uppercase_normalizer" + }, + "request_ip_region": { + "type": "keyword", + "normalizer": "uppercase_normalizer" + }, + "request_method": { + "type": "keyword", + "normalizer": "uppercase_normalizer" + }, + "request_origin": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_path": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_referer": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_scheme": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_size": { + "type": "integer" + }, + "request_url_hierarchy_level0": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_url_hierarchy_level1": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_url_hierarchy_level2": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_url_hierarchy_level3": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_url_hierarchy_level4": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_url_hierarchy_level5": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_url_hierarchy_level6": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_url_query": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_user_agent": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_user_agent_family": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "request_user_agent_type": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_age": { + "type": "integer" + }, + "response_cache": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_content_encoding": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_content_length": { + "type": "integer" + }, + "response_content_type": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_server": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "response_size": { + "type": "integer" + }, + "response_status": { + "type": "short" + }, + "response_time": { + "type": "integer" + }, + "response_transfer_encoding": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "user_id": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "user_email": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + }, + "user_registration_source": { + "type": "keyword", + "normalizer": "lowercase_normalizer" + } + } + } + } + } + } +] diff --git a/config/test.yml b/config/test.yml index d3a811824..8f94cf2b4 100644 --- a/config/test.yml +++ b/config/test.yml @@ -12,25 +12,15 @@ nginx: # hard-coded for test purposes so we have a more stable baseline and ensure # our tests always run with multiple workers. workers: 2 + # Disable log output buffering for immediate output in tests. access_log_options: + # Increase error log verbosity in tests. + error_log_level: notice proxy_connect_timeout: 10 proxy_read_timeout: 10 proxy_send_timeout: 10 - dyups: - port: 13005 trafficserver: port: 13009 - autoconf_port: 13007 - management_port: 13008 - embedded_server_config: - records: - # In the test environment disable fuzzy revalidation since it makes - # things difficult to test against. For production, it seems like a - # decent idea to keep the feature enabled, but in the test environment, - # it means that 0.5% of requests might not behave as expected (since they - # will be re-fetched ahead of their actual TTL). - # See: https://docs.trafficserver.apache.org/en/latest/admin/http-proxy-caching.en.html?highlight=fuzz#fuzzy-revalidation - - CONFIG proxy.config.http.cache.fuzz.time INT 0 api_server: port: 13010 web: @@ -39,17 +29,17 @@ web: admin: auth_strategies: facebook: - client_id: test_fake - client_secret: test_fake + client_id: test_fake_id + client_secret: test_fake_secret github: - client_id: test_fake - client_secret: test_fake + client_id: test_fake_id + client_secret: test_fake_secret gitlab: - client_id: test_fake - client_secret: test_fake + client_id: test_fake_id + client_secret: test_fake_secret google: - client_id: test_fake - client_secret: test_fake + client_id: test_fake_id + client_secret: test_fake_secret ldap: options: host: 127.0.0.1 @@ -63,8 +53,6 @@ web: port: 13102 contact_form_email: default-test-contact-email@example.com router: - api_backends: - port: 13011 trusted_proxies: - 192.168.12.0/23 - 10.10.10.10 @@ -89,14 +77,6 @@ elasticsearch: transport: tcp: port: 13003 - discovery: - zen: - ping: - multicast: - enabled: false - index: - number_of_shards: 1 - number_of_replicas: 0 unbound: port: 13100 control_port: 13101 @@ -109,3 +89,9 @@ openldap: port: 13104 apiSettings: require_https: optional +_test_config: + default_null_override_hash: + foo: bar + default_null_override_string: foobar + default_empty_hash_override_hash: + baz: qux diff --git a/configure b/configure index d13ddffc1..1ea985c19 100755 --- a/configure +++ b/configure @@ -1,307 +1,48 @@ -#!/bin/sh +#!/usr/bin/env bash -# Autotools-style (./configure) wrapper for CMake -# -# -# *** IMPORTANT *** -# -# You must include the GNUInstallDirs module (which comes with -# CMake) in your project. Just put "include (GNUInstallDirs)" in -# you CMakeLists.txt and you should be good. -# -# This script was originally written for Squash -# by Evan Nemerson -# , but has been spun off into a separate -# repository. Please feel free to copy it into your own repository, -# though I would appreciate it if you would post improvements, bugs, -# feature requests, etc. to the issue tracker at -# . -# -# To the extent possible under law, the author(s) hereby waive all -# copyright and related or neighboring rights to this work. For -# details, see +set -e -u -TOP_SRCDIR="$(dirname $0)" -CMAKE_CMD="cmake ${TOP_SRCDIR}" +prefix="/opt/api-umbrella" -BUILD_TYPE="Debug" -PREFIX=/usr/local -LIBDIR= -CMAKE_ARGS= +usage() { + cat <&2 +Configuration: -h, --help display this help and exit - --pass-thru pass remaining arguments through to CMake +Installation directories: --prefix=PREFIX install architecture-independent files in PREFIX - [$PREFIX] -EOF - - first=y - for varstring in ${ENABLE_VARS}; do - if [ $first = 'y' ]; then - echo "" - first=n - fi - extract_var_string $(echo "${varstring}" | tr '|' ' ') - var_doc_name="ENABLE_${VAR_UC_NAME}_DOC" - eval "docstring=\$$var_doc_name" - if [ "x${docstring}" = "x" ]; then - printf " --enable-%-14s enable %s support\n" "${VAR_NAME}" "$(echo -n "${VAR_NAME}" | tr '-' ' ')" - else - printf " --enable-%-14s %s\n" "${VAR_NAME}" "$docstring" - fi - done - - first=y - for varstring in ${DISABLE_VARS}; do - if [ $first = 'y' ]; then - echo "" - first=n - fi - extract_var_string $(echo "${varstring}" | tr '|' ' ') - var_doc_name="DISABLE_${VAR_UC_NAME}_DOC" - eval "docstring=\$$var_doc_name" - if [ "x${docstring}" = "x" ]; then - printf " --disable-%-13s disable %s support\n" "${VAR_NAME}" "$(echo -n "${VAR_NAME}" | tr '-' ' ')" - else - printf " --disable-%-13s %s\n" "${VAR_NAME}" "$docstring" - fi - done - - first=y - for varstring in ${WITH_VARS}; do - if [ $first = 'y' ]; then - echo "" - first=n - fi - extract_var_string $(echo "${varstring}" | tr '|' ' ') - var_doc_name="WITH_${VAR_UC_NAME}_DOC" - eval "docstring=\$$var_doc_name" - paraminfo="${VAR_NAME}=${VAR_VALUE}" - if [ "x${docstring}" = "x" ]; then - printf " --with-%-16s enable %s support\n" "$paraminfo" "$(echo -n "${VAR_NAME}" | tr '-' ' ')" - else - printf " --with-%-16s %s\n" "$paraminfo" "$docstring" - fi - done - - exit 0 + [$prefix] +EOS + exit 0 } -while [ $# != 0 ]; do - case "$1" in - "--prefix="*) - PREFIX="${1#*=}";; - "--prefix") - PREFIX="${2}"; shift;; - "--bindir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_BINDIR=$(quote "${1#*=}")";; - "--bindir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_BINDIR=$(quote "$2")"; shift;; - "--sbindir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_SBINDIR=$(quote "${1#*=}")";; - "--sbindir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_SBINDIR=$(quote "$2")"; shift;; - "--libexecdir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_LIBEXECDIR=$(quote "${1#*=}")";; - "--libexecdir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_LIBEXECDIR=$(quote "$2")"; shift;; - "--sysconfdir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_SYSCONFDIR=$(quote "${1#*=}")";; - "--sysconfdir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_SYSCONFDIR=$(quote "$2")"; shift;; - "--sharedstatedir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_SHAREDSTATEDIR=$(quote "${1#*=}")";; - "--sharedstatedir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_SHAREDSTATEDIR=$(quote "$2")"; shift;; - "--localstatedir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_LOCALSTATEDIR=$(quote "${1#*=}")";; - "--localstatedir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_LOCALSTATEDIR=$(quote "$2")"; shift;; - "--libdir="*) - LIBDIR="${1#*=}";; - "--libdir") - LIBDIR="${2}"; shift;; - "--includedir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_INCLUDEDIR=$(quote "${1#*=}")";; - "--includedir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_INCLUDEDIR=$(quote "$2")"; shift;; - "--oldincludedir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_OLDINCLUDEDIR=$(quote "${1#*=}")";; - "--oldincludedir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_OLDINCLUDEDIR=$(quote "$2")"; shift;; - "--datarootdir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_DATAROOTDIR=$(quote "${1#*=}")";; - "--datarootdir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_DATAROOTDIR=$(quote "$2")"; shift;; - "--datadir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_DATADIR=$(quote "${1#*=}")";; - "--datadir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_DATADIR=$(quote "$2")"; shift;; - "--infodir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_INFODIR=$(quote "${1#*=}")";; - "--infodir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_INFODIR=$(quote "$2")"; shift;; - "--localedir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_LOCALEDIR=$(quote "${1#*=}")";; - "--localedir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_LOCALEDIR=$(quote "$2")"; shift;; - "--mandir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_MANDIR=$(quote "${1#*=}")";; - "--mandir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_MANDIR=$(quote "$2")"; shift;; - "--docdir="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_DOCDIR=$(quote "${1#*=}")";; - "--docdir") - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_INSTALL_DOCDIR=$(quote "$2")"; shift;; - - "CC="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_C_COMPILER=$(quote "${1#*=}")";; - "CXX="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_CXX_COMPILER=$(quote "${1#*=}")";; - "CFLAGS="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_C_FLAGS=$(quote "${1#*=}")";; - "CXXFLAGS="*) - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_CXX_FLAGS=$(quote "${1#*=}")";; - "LDFLAGS="*) - LDFLAGS="$LDFLAGS ${1#*=}";; - - "--help") - print_help;; - "-h") - print_help;; - - # This flag is the only one which may be a bit surprising to - # people. Autotools always builds with debugging symbols enabled - # (AFAIK), but for cmake you have to do -DCMAKE_BUILD_TYPE=Debug. - # Unfortunately this can change other things as well, so although - # I realize there is no --disable-debug flag I thought it would be - # prudent to support one here. - "--disable-debug") - BUILD_TYPE="Release";; - - "--pass-thru") - shift; - while [ $# != 0 ]; do - CMAKE_ARGS="$CMAKE_ARGS $(quote "${1}")"; - shift; - done;; - - "--enable-"*) - set_config_var "$1" - ;; - - "--disable-"*) - set_config_var "$1" - ;; - - "--with-"*) - name=$(echo "${1#--with-}" | awk '{split($1,v,"="); print v[1]}') - case "${1}" in - "--with-${name}="*) - set_config_var "--with-${name}" "${1#--with-${name}=}";; - "--with-${name}") - set_config_var "$1" "$2"; - shift;; - esac - ;; - - *) - echo "$0: error: unrecognized option: \`$1'" >&2 - echo "Try \`$0 --help' for more information" >&2 - exit -1 - esac; - shift +while [ $# -gt 0 ]; do + key="$1" + case $key in + --prefix=*) + prefix="${key#*=}" + shift + ;; + --help|-h) + usage + shift + ;; + *) + echo "configure: error: unrecognized option: '$key'" + echo "Try './configure --help' for more information" + exit 1 + ;; + esac done -if [ "x${LIBDIR}" = "x" ]; then - LIBDIR="${PREFIX}/lib" +if [ -z "$prefix" ]; then + echo "configure: error: missing argument to --prefix" + exit 1 fi -# Unlike CFLAGS/CXXFLAGS/CC/CXX, LDFLAGS isn't handled by CMake, so we -# need to parse it here. -if [ "x${LDFLAGS}" != "x" ]; then - for varname in EXE MODULE SHARED STATIC; do - CMAKE_ARGS="$CMAKE_ARGS -DCMAKE_${varname}_LINKER_FLAGS=$(quote "$LDFLAGS")" - done -fi - -"$TOP_SRCDIR/build/scripts/download_cmake" -build_dir="$(pwd)" -PATH="$build_dir/build/work/cmake/bin:$PATH" -export PATH - -eval "cmake ${TOP_SRCDIR} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${PREFIX} ${CMAKE_ARGS}" +echo "configure: creating Makefile" +sed -e "s|@prefix@|$prefix|g" < Makefile.in > Makefile diff --git a/deploy/config/deploy.rb b/deploy/config/deploy.rb index c6de0c5b5..1497b95ef 100644 --- a/deploy/config/deploy.rb +++ b/deploy/config/deploy.rb @@ -10,7 +10,7 @@ # repo out directly on the server). This allows for the deployments to work # from custom forks without having to update the URL. set :scm, :rsync -set :repo_url, "file://#{File.expand_path("../../../", __FILE__)}" +set :repo_url, "file://#{File.expand_path("../..", __dir__)}" # Default branch is :master # ask :branch, `git rev-parse --abbrev-ref HEAD`.chomp diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml new file mode 100644 index 000000000..6d7e7fe43 --- /dev/null +++ b/docker-compose-dev.yml @@ -0,0 +1,42 @@ +version: "3.5" + +services: + mongo: + image: mongo:3.6 + networks: + main: + aliases: + - mongodb.docker + volumes: + - ./mongo-data:/data/db + + umbrella_elasticsearch: + image: elasticsearch:2.4 + volumes: + - ./umbrella-elasticsearch:/usr/share/elasticsearch/data + command: elasticsearch -Des.index.max_result_window=50000 + networks: + main: + aliases: + - elasticsearch.docker + + umbrella: + # Image build with Dockerfile-build or Dockerfile-dev-build + image: umbrella-dev + depends_on: + - mongo + - umbrella_elasticsearch + ports: + - 8443:8443 + volumes: + - ./dep-config:/etc/api-umbrella + # Mount the source code to allow live software updates while developing + - ./src/:/app/src + networks: + main: + aliases: + - umbrella.docker + +networks: + main: + external: true diff --git a/docker-compose.yml b/docker-compose.yml index 2c2a50f10..66ec6a9bc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,10 +7,6 @@ services: volumes: - .:/app - build_cache:/build - tmpfs: - # Mount the ember-cli tmp directory as a tmpfs partition for better - # performance. - - /app/src/api-umbrella/admin-ui/tmp:rw,size=128m,mode=1777 environment: HTTP_PORT: 8888 HTTPS_PORT: 8443 @@ -22,6 +18,8 @@ services: aliases: - umbrella.docker + sysctls: + - net.ipv6.conf.all.disable_ipv6=0 networks: main: external: true diff --git a/docker/Dockerfile b/docker/Dockerfile index 745e14925..f8dc55d3d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -4,7 +4,7 @@ ENV API_UMBRELLA_VERSION 0.14.4-1~jessie # Install API Umbrella RUN echo "deb http://dl.bintray.com/nrel/api-umbrella-debian jessie main" >> /etc/apt/sources.list.d/api-umbrella.list -RUN apt-get update && apt-get -y --allow-unauthenticated install api-umbrella +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends --allow-unauthenticated install api-umbrella # Define mountable directories VOLUME ["/etc/api-umbrella", "/opt/api-umbrella/var/db", "/opt/api-umbrella/var/log"] diff --git a/docker/dev/docker-entrypoint b/docker/dev/docker-entrypoint new file mode 100755 index 000000000..579bf23f2 --- /dev/null +++ b/docker/dev/docker-entrypoint @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -e -u -x + +# Create tests file +echo "#!/usr/bin/env bash" > test.sh +echo "unbuffer ./tasks/test/default" >> test.sh +chmod +x test.sh + +mkdir -p /etc/api-umbrella +{ + echo "app_env: production" + echo "http_port: $HTTP_PORT" + echo "https_port: $HTTPS_PORT" +} > /etc/api-umbrella/api-umbrella.yml + +mkdir -p /build/.task +ln -snf /build/.task /app/.task + +mkdir -p /build/build/work/stage/opt/api-umbrella/var/log +ln -snf /build/build/work/stage/opt/api-umbrella /opt/api-umbrella +ln -snf /build/build/work/stage/opt/api-umbrella/var/log /var/log/api-umbrella +ln -snf /build/build/work /app/build/work + +mkdir -p /build/test/tmp/run +mkdir -p /app/test/tmp +ln -snf /build/test/tmp/run /app/test/tmp/run + +mkdir -p /app/build/work/tasks/app-deps/admin-ui/yarn/_persist/node_modules +ln -snf /app/build/work/tasks/app-deps/admin-ui/yarn/_persist/node_modules /app/src/api-umbrella/admin-ui/node_modules + +mkdir -p /app/build/work/tasks/test-deps/bundle/_persist/.bundle +ln -snf /app/build/work/tasks/test-deps/bundle/_persist/.bundle /app/.bundle + +mkdir -p /app/build/work/tasks/app-deps/web-app/bundle/_persist/.bundle +ln -snf /app/build/work/tasks/app-deps/web-app/bundle/_persist/.bundle /app/src/api-umbrella/web-app/.bundle + +mkdir -p /app/src/api-umbrella/admin-ui/tmp +chmod 1777 /app/src/api-umbrella/admin-ui/tmp + +exec "$@" diff --git a/docker/dev/docker-start b/docker/dev/docker-start index b04f26356..f378f9a70 100755 --- a/docker/dev/docker-start +++ b/docker/dev/docker-start @@ -2,36 +2,5 @@ set -e -u -x -mkdir -p /etc/api-umbrella -{ - echo "app_env: development" - echo "http_port: $HTTP_PORT" - echo "https_port: $HTTPS_PORT" - echo "gatekeeper:" - echo " api_key_cache: false" - echo " default_idp:" - echo " backend_name: fiware-oauth2" - echo " host: http://idm.docker:8000" - echo "apiSettings:" - echo " error_data:" - echo " token_not_supported:" - echo " status_code: 403" - echo " code: TOKEN_NOT_SUPPORTED" - echo " message: The accessed API do not support authorization based on a token issued by an external identity provider, use your API key instead" -} > /etc/api-umbrella/api-umbrella.yml - -ln -snf /build/build/work/stage/opt/api-umbrella /opt/api-umbrella -ln -snf /build/build/work/stage/opt/api-umbrella/var/log /var/log/api-umbrella -ln -snf /build/build/work ./build/work -ln -snf /build/CMakeFiles ./CMakeFiles -ln -snf /build/CMakeCache.txt ./CMakeCache.txt -ln -snf /build/Makefile ./Makefile -mkdir -p /build/test/tmp/run -mkdir -p ./test/tmp -ln -snf /build/test/tmp/run ./test/tmp/run - -(cd /build && env PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" make) - -chmod 1777 /app/src/api-umbrella/admin-ui/tmp - +make api-umbrella run diff --git a/docker/docker-entrypoint b/docker/docker-entrypoint new file mode 100755 index 000000000..ed141696c --- /dev/null +++ b/docker/docker-entrypoint @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -e -u -x + +mkdir -p /etc/api-umbrella + +mkdir -p /build/.task +ln -snf /build/.task /app/.task + +mkdir -p /build/build/work/stage/opt/api-umbrella/var/log +ln -snf /build/build/work/stage/opt/api-umbrella /opt/api-umbrella +ln -snf /build/build/work/stage/opt/api-umbrella/var/log /var/log/api-umbrella +ln -snf /build/build/work /app/build/work + +mkdir -p /build/test/tmp/run +mkdir -p /app/test/tmp +ln -snf /build/test/tmp/run /app/test/tmp/run + +mkdir -p /app/build/work/tasks/app-deps/admin-ui/yarn/_persist/node_modules +ln -snf /app/build/work/tasks/app-deps/admin-ui/yarn/_persist/node_modules /app/src/api-umbrella/admin-ui/node_modules + +mkdir -p /app/build/work/tasks/test-deps/bundle/_persist/.bundle +ln -snf /app/build/work/tasks/test-deps/bundle/_persist/.bundle /app/.bundle + +mkdir -p /app/build/work/tasks/app-deps/web-app/bundle/_persist/.bundle +ln -snf /app/build/work/tasks/app-deps/web-app/bundle/_persist/.bundle /app/src/api-umbrella/web-app/.bundle + +mkdir -p /app/src/api-umbrella/admin-ui/tmp +chmod 1777 /app/src/api-umbrella/admin-ui/tmp + +exec "$@" diff --git a/docs/developer/analytics-architecture.md b/docs/developer/analytics-architecture.md index e91e4ad54..c76df8c79 100644 --- a/docs/developer/analytics-architecture.md +++ b/docs/developer/analytics-architecture.md @@ -16,12 +16,8 @@ To explain each step: - It can transform the data and send it to multiple different endpoints. - The storage database stores the raw analytics data for further querying or processing. -API Umbrella supports different analytics databases: - ## Elasticsearch -Suitable for small to medium amounts of historical analytics data. *(TODO: Provide more definitive guidance on what small/medium/large amounts are)* - ### Ingest Data is logged directly to Elasticsearch from rsyslog: @@ -41,132 +37,3 @@ The analytic APIs in the web application directly query Elasticsearch: ``` [api-umbrella-web-app] => [Elasticsearch] ``` - -## PostgreSQL - -**TODO: The PostgreSQL adapter doesn't currently exist, but the idea is to leverage the same SQL framework built for Kylin.** - -Suitable for small amounts of historical analytics data, or small to medium amounts of data with a columnar storage extension. *(TODO: Provide more definitive guidance on what small/medium/large amounts are)* - -### Ingest - -Data is logged directly to PostgreSQL from rsyslog: - -``` -[nginx] ====> [rsyslog] ===> [PostgreSQL] - JSON SQL -``` - -- rsyslog buffers and sends data to PostgreSQL as individual inserts. -- rsyslog's ompgsql output module is used. -- If rsyslog supports batched transactions in the future, we should switch to that: [rsyslog#895](https://github.com/rsyslog/rsyslog/issues/895) - -### Querying - -The analytic APIs in the web application directly query PostgreSQL: - -``` -[api-umbrella-web-app] ===> [PostgreSQL] - SQL -``` - -### PostgreSQL: Columnar Storage - -For better analytics performance with larger volumes of analytics data, you can continue to use the PostgreSQL adapter, but with a compatible column-based variant: - -- [cstore_fdw](https://github.com/citusdata/cstore_fdw) -- [Amazon Redshift](https://aws.amazon.com/redshift/) - -When these are used, the SQL table design and process remains the same, only the underlying table storage is changed for better analytic query performance. - -## Kylin - -Suitable for large amounts of historical analytics data. *(TODO: Provide more definitive guidance on what small/medium/large amounts are)* - -This is the most complicated setup, but it allows for vastly improved querying performance when dealing with large amounts of historical data. This is achieved by using [Kylin](http://kylin.apache.org) to pre-compute common aggregate totals. By pre-computing common aggregations, less hardware is needed than would otherwise be needed to quickly answer analytics queries over large amounts of data. Under this approach, analytics data may not be immediately available for querying, since additional processing is required. - -### Ingest - -During ingest, there are several concurrent processes that play a role: - -``` -[nginx] ====> [rsyslog] ====> [Kafka] ====> [Flume] ====> [HDFS - JSON (temp)] - JSON JSON JSON JSON -``` - -``` -[HDFS - JSON (temp)] => [API Umbrella Live Processor] => [Hive - ORC] -``` - -``` -[Hive - ORC] => [API Umbrella Kylin Refresher] => [Kylin] -``` - -- rsyslog buffers and sends JSON messages to Kafka using the [omkafka](http://www.rsyslog.com/doc/v8-stable/configuration/modules/omkafka.html) output module. - - Kafka is used as an intermediate step as a reliable way to get messages in order to Flume, but primarily Kafka is being used because that's what Kylin's future [streaming feature](http://kylin.apache.org/blog/2016/02/03/streaming-cubing/) will require (so it seemed worth getting in place now). -- Flume takes messages off the Kafka queue and appends them to a gzipped JSON file stored inside Hadoop (HDFS). - - The JSON files are flushed to HDFS every 15 seconds, and new files are created for each minute. - - The per-minute JSON files are partitioned by the request timestamp and not the timestamp of when Flume is processing the message. This means Flume could be writing to a file from previous minutes if it's catching up with a backlog of data. - - Kafka's stronger in-order handling of messages should ensure that the per-minute JSON files are written in order, and skipping between minutes should not be likely (although possible if an nginx server's clock is severely skewed or an nginx server goes offline, but still has queued up messages that could be sent if it rejoins later). - - Flume plays a very similar role to rsyslog, but we use it because it has the best integration with the Hadoop ecosystem and writing to HDFS (I ran into multiple issues with rsyslog's native omhdfs and omhttpfs modules). -- The API Umbrella Live Processor task determines when a per-minute JSON file hasn't been touched in more than 1 minute, and then copies the data to the ORC file for permanent storage and querying in the Hive table. - - The live data should usually make it's way to the permanent ORC storage within 2-3 minutes. - - The ORC data is partitioned by day. - - The data is converted from JSON to ORC using a Hive SQL command. Each minute of data is appended as a new ORC file within the overall ORC daily partition (which Hive simply treats as a single daily partition within the overall logs table). - - Since the data is only appended, the same minute cannot be processed twice, which is why we give a minute buffer after the JSON file has ceased writing activity to convert it to ORC. - - The ORC file format gives much better compression and querying performance than storing everything in JSON. - - If a new ORC file is created for a new day, the partition will be added to the Hive table. - - At the end of each day, overwrite the daily ORC file with a new, compacted file from the original JSON data. Writing the full day at once provides better querying performance than the many per-minute ORC files. By basing this daily file on the original JSON data, it also alleviates any rare edge-cases where the per-minute appender missed data. - - Automatically remove old JSON minute data once it's no longer needed. -- The API Umbrella Kylin Refresher task is responsible for triggering Kylin builds to updated the pre-aggregated data. - - At the end of each day, after writing the compacted ORC file for the full day, we then trigger a Kylin build for the most recent day's data. - -This setup is unfortunately complicated with several moving pieces. However, there are several things that could potentially simplify this setup quite a bit in the future: - -- [Kylin Streaming](http://kylin.apache.org/blog/2016/02/03/streaming-cubing/): This would eliminate our need to constantly refresh Kylin throughout the day, and reduce the amount of time it would take live data to become available in Kylin's pre-aggregated results. This feature available as a prototype in Kylin 1.5, but we're still on 1.2, and we'll be waiting for this to stabilize and for more documentation to come out. But basically, this should just act as another consumer of the Kafka queue, and then it would handle all the details of getting the data into Kylin. -- [Flume Hive Sink](https://flume.apache.org/FlumeUserGuide.html#hive-sink): Even with Kylin streaming support, we will likely still need our own way to get the live data into the ORC-backed Hive table. Flume's Hive Sink offers a way to directly push data from Flume into a ORC table. Currently marked as a preview feature, I ran into memory growth and instability issues in my attempts to use it, but if this proves stable in the future, it could be a much easier path to populating the ORC tables directly and get rid of the need for temporary JSON (along with the edge conditions those bring). -- [Kylin Hour Partitioning](https://issues.apache.org/jira/browse/KYLIN-1427): A possible shorter-term improvement while waiting for Kylin streaming is the ability to refresh Kylin by hour partitions. This would be more efficient than our full day refreshes currently used. This is currently implemented in v1.5.0, but we first need to upgrade to 1.5 (we're holding back at 1.2 due to some other issues), and then [KYLIN-1513](https://issues.apache.org/jira/browse/KYLIN-1513) would be good to get fixed before. - -### Querying - -The analytic APIs in the web application query Kylin or [PrestoDB](https://prestodb.io) using SQL statements: - -``` - /==> [Kylin] ====> [HBase Aggregates] - / -[api-umbrella-web-app] ===> - SQL \ - \==> [PrestoDB] => [Hive ORC Tables] -``` - -- Queries are attempted against Kylin first, since Kylin will provide the fastest answers from it's pre-computed aggregates. - - Kylin will be unable to answer the query if the query involves dimensions that have not been pre-computed. - - We've attempted to design the Kylin cubes with the dimensions that are involved in the most common queries. These are currently: - - `timestamp_tz_year` - - `timestamp_tz_month` - - `timestamp_tz_week` - - `timestamp_tz_date` - - `timestamp_tz_hour` - - `request_url_host` - - `request_url_path_level1` - - `request_url_path_level2` - - `request_url_path_level3` - - `request_url_path_level4` - - `request_url_path_level5` - - `request_url_path_level6` - - `user_id` - - `request_ip` - - `response_status` - - `denied_reason` - - `request_method` - - `request_ip_country` - - `request_ip_region` - - `request_ip_city` - - We don't add all the columns/dimensions to the Kylin cubes, since each additional dimension exponentially increases the amount of data Kylin has to pre-compute (which can significantly increase processing time and storage). - - Data must be processed into Kylin for it to be part of Kylin's results, so the results will typically lag 30-60 minutes behind live data. -- If Kylin fails for any reason (the query involves a column we haven't precomputed or Kylin is down), then we perform the same query against PrestoDB. This queries the underlying ORC tables stored in Hive (which is the same raw data Kylin bases its data cubes on). - - PrestoDB is used to provide an ANSI SQL layer on top of Hive. This should provide better compatibility with the SQL queries we're sending to Kylin, since both Kylin and PrestoDB aim for ANSI SQL compatibility (unlike Hive, which uses a different SQL-like HiveQL). - - PrestoDB also offers better performance (significant in some cases) for our SQL queries rather than querying Hive directly. PrestoDB has also been fairly optimized for querying ORC tables. - - Queries hitting PrestoDB will be slower than Kylin-answered queries. Query times vary primary depending on how much data is being queried, but response times may range from 5 seconds to multiple minutes. - - Data must be processed into the ORC-backed Hive table for it to be part of PrestoDB's results, so results will typically lag 2-3 minutes behind live data (and therefore differ from Kylin results). - - *TODO: Currently there's a 60 second timeout on PrestoDB queries to prevent long-running queries from piling up and hogging resources. However, if we find that people need to run longer-running queries, we can adjust this. We'll also need to adjust the default [60 second proxy timeouts](https://github.com/NREL/api-umbrella/blob/v0.11.0/config/default.yml#L21-L23).* diff --git a/docs/developer/compiling-from-source.md b/docs/developer/compiling-from-source.md index 21c6a2fff..627069fdd 100644 --- a/docs/developer/compiling-from-source.md +++ b/docs/developer/compiling-from-source.md @@ -6,7 +6,7 @@ Installing from a [binary package](../getting-started.html#installation) is reco - 64bit Linux distribution - It should be possible to run against other 64bit *nix operating systems, but our build script currently has some hard-coded assumptions to a 64bit linux environment. [File an issue](https://github.com/NREL/api-umbrella/issues/new) if you'd like to see other operating systems supported. -- Dependencies can automatically be installed for supported distributions by running the `./build/scripts/install_build_dependencies` script. For unsupported distributions, view the `./build/package_dependencies.sh` file for a list of required packages. +- Dependencies can automatically be installed for supported distributions by running the `./tasks/install-system-build-dependencies` script. For unsupported distributions, view the `./build/package_dependencies.sh` file for a list of required packages. ## Compiling & Installing @@ -14,7 +14,7 @@ Installing from a [binary package](../getting-started.html#installation) is reco $ curl -OLJ https://github.com/NREL/api-umbrella/archive/v0.14.4.tar.gz $ tar -xvf api-umbrella-0.14.4.tar.gz $ cd api-umbrella-0.14.4 -$ sudo ./build/scripts/install_build_dependencies +$ sudo ./tasks/install-system-build-dependencies $ ./configure $ make $ sudo make install diff --git a/docs/developer/deploying.md b/docs/developer/deploying.md index 9541c5354..aea235e9c 100644 --- a/docs/developer/deploying.md +++ b/docs/developer/deploying.md @@ -31,7 +31,7 @@ On each server you wish to deploy to, you must setup SSH keys so that you can de ### Install Build Dependencies -On each server you wish to deploy to, you must install the system packages needed for building dependencies (for example, make, gcc, etc). This can be automated through the `build/scripts/install_build_dependencies` shell script: +On each server you wish to deploy to, you must install the system packages needed for building dependencies (for example, make, gcc, etc). This can be automated through the `tasks/install-system-build-dependencies` shell script: - On each server: @@ -39,7 +39,7 @@ On each server you wish to deploy to, you must install the system packages neede $ curl -OLJ https://github.com/NREL/api-umbrella/archive/master.tar.gz $ tar -xvf api-umbrella-master.tar.gz $ cd api-umbrella-master - $ sudo ./build/scripts/install_build_dependencies + $ sudo ./tasks/install-system-build-dependencies ``` ## Deploying diff --git a/docs/developer/dev-setup.md b/docs/developer/dev-setup.md index 5dfa3fc2f..e19ff7def 100644 --- a/docs/developer/dev-setup.md +++ b/docs/developer/dev-setup.md @@ -1,78 +1,52 @@ # Development Setup -The easiest way to get started with API Umbrella development is to use [Vagrant](http://www.vagrantup.com/) to setup a local development environment. +The easiest way to get started with API Umbrella development is to use [Docker](https://www.docker.com) to setup a local development environment. ## Prerequisites - 64bit CPU - the development VM requires an 64bit CPU on the host machine -- [VirtualBox](https://www.virtualbox.org/wiki/Downloads) (version 4.3 or higher) -- [Vagrant](https://www.vagrantup.com/downloads.html) (version 1.6 or higher) -- [ChefDK](https://downloads.chef.io/chef-dk/) (version 0.10 or higher) -- NFS: For Mac OS X or Linux host machines only: - - Mac OS X: Already installed and running - - Ubuntu: `sudo apt-get install nfs-kernel-server nfs-common portmap` +- [Docker](https://www.docker.com/get-started) ## Setup -After installing VirtualBox and Vagrant, follow these steps: +After installing Docker, follow these steps: ```sh -# Install the required Vagrant plugins -$ vagrant plugin install vagrant-berkshelf - # Get the code and spinup your development VM $ git clone https://github.com/NREL/api-umbrella.git $ cd api-umbrella -$ vagrant up # This step compiles API Umbrella from source, so the first time - # make take 30-40 minutes. +$ docker-compose up ``` -Assuming all goes smoothly, you should be able to see the homepage at [http://10.10.33.2/](http://10.10.33.2/). - -If you run into issues when running `vagrant up`, try running `vagrant provision` once to see if the error reoccurs. This will pickup with the setup process from the last failure point, which can sometimes help resolve temporary issues. +Assuming all goes smoothly, you should be able to see the homepage at [https://localhost:8101/](https://localhost:8101/). You will need to need to accept the self-signed SSL certificate for localhost in order to access the development environment. -If you're still having any difficulties getting the Vagrant environment setup, then open an [issue](https://github.com/NREL/api-umbrella/issues). +If you're having any difficulties getting the development environment setup, then open an [issue](https://github.com/NREL/api-umbrella/issues). ## Directory Structure A quick overview of some of the relevant directories for development: +- `src/api-umbrella/admin-ui`: The admin user interface which utilizes the administrative APIs provided by the web-app. - `src/api-umbrella/cli`: The actions behind the `api-umbrella` command line tool. - `src/api-umbrella/proxy`: The custom reverse proxy where API requests are validated before being allowed to the underlying API backend. -- `src/api-umbrella/web-app`: Provides the admin tool and APIs. -- `src/api-umbrella/web-app/spec`: Tests for the admin tool and APIs. +- `src/api-umbrella/web-app`: Provides the public and administrative APIs. - `test`: Proxy tests and integration tests for the entire API Umbrella stack. ## Making Code Changes -This development VM runs the various components in "development" mode, which typically means any code changes you make will immediately be reflected. However, this does mean this development VM will run API Umbrella slower than in production. +This development environment runs the various components in "development" mode, which typically means any code changes you make will immediately be reflected. However, this does mean this development environment will run API Umbrella slower than in production. While you can typically edit files and see your changes, for certain types of application changes, you may need to restart the server processes. There are two ways to restart things if needed: ```sh -# These commands must be executed *inside* your Vagrant VM: -$ vagrant ssh +# Quick: Reload most server processes by executing a reload command: +docker-compose exec app api-umbrella reload -# Quick: This should restart most server processes you'll need as a developer, -# but this doesn't restart everything: -$ sudo /etc/init.d/api-umbrella reload - -# Slow: Restarts everything: -$ sudo /etc/init.d/api-umbrella restart +# Slow: Fully restart everything: +docker-compose stop +docker-compose up ``` ## Writing and Running Tests See the [testing section](testing.html) for more information about writing and running tests. - -## Customizing Your VM - -The following environment variables can be set prior to running `vagrant up` if you wish to tune the local VM (for example, to give it more or less memory, pick a different IP address, or use a different base box): - -``` -API_UMBRELLA_VAGRANT_BOX=nrel/CentOS-6.7-x86_64 -API_UMBRELLA_VAGRANT_MEMORY=2048 -API_UMBRELLA_VAGRANT_CORES=2 -API_UMBRELLA_VAGRANT_IP=10.10.33.2 -API_UMBRELLA_VAGRANT_NFS=true -``` diff --git a/docs/developer/testing.md b/docs/developer/testing.md index 3def748db..5aa17eb18 100644 --- a/docs/developer/testing.md +++ b/docs/developer/testing.md @@ -12,11 +12,10 @@ API Umbrella's test suite uses Ruby's [minitest](https://github.com/seattlerb/mi ## Running Tests -Assuming you have a [Vagrant development environment](dev-setup.html), you can run all the tests with: +Assuming you have a [Docker development environment](dev-setup.html), you can run all the tests with: ```sh -$ cd /vagrant -$ make test +docker-compose run --rm app make test ``` ### Running Individual Tests @@ -24,7 +23,5 @@ $ make test If you'd like to run individual tests, rather than all the tests, there are a few different ways to do that: ```sh -# Run individual files or tests within the web-app test suite: -$ cd /vagrant -$ ruby test/apis/v1/admins/test_create.rb +docker-compose run --rm app bundle exec minitest test/apis/v1/admins/test_create.rb ``` diff --git a/docs/index.rst b/docs/index.rst index 1706794f6..1cc50e897 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,7 +47,6 @@ API Umbrella is an open source API management platform for exposing web service server/listen-ports server/logs server/db-config - server/analytics-storage-adapters .. toctree:: :caption: For API Consumers diff --git a/docs/server/analytics-storage-adapters.md b/docs/server/analytics-storage-adapters.md deleted file mode 100644 index a724811d7..000000000 --- a/docs/server/analytics-storage-adapters.md +++ /dev/null @@ -1,37 +0,0 @@ -# Analytics Storage Adapters - -API Umbrella can store analytics data in different types of databases, depending on your performance needs and volume of metrics. The adapter can be picked by setting the `analytics.adapter` option inside the `/etc/api-umbrella/api-umbrella.yml` configuration file. - -## Elasticsearch - -`analytics.adapter: elasticsearch` - -- Currently the default analytics store. -- Suitable for small to medium amounts of historical analytics data. *(TODO: Provide more definitive guidance on what small/medium/large amounts are)* -- API Umbrella ships with with a default ElasticSearch database that can be used or any ElasticSearch 1.7+ cluster can be used. - -## Kylin - -`analytics.adapter: kylin` - -- Suitable for large amounts of historical analytics data. *(TODO: Provide more definitive guidance on what small/medium/large amounts are)* -- Requires a functional Hadoop environment compatible with Kylin 1.2 ([Hortonworks Data Platform (HDP) 2.2](http://hortonworks.com/products/releases/hdp-2-2/) is recommended). -- Requires Kafka (can be enabled as part of HDP). -- Requires the optional `api-umbrella-hadoop-analytics` package to be installed on the analytics database server. *(TODO: Link to hadoop-analytics package downloads once built)* - -## PostgreSQL - -`analytics.adapter: postgresql` - -**TODO: The PostgreSQL adapter doesn't currently exist, but the idea is to leverage the same SQL framework built for Kylin.** - -- Suitable for small amounts of historical analytics data. *(TODO: Provide more definitive guidance on what small/medium/large amounts are)* - -### PostgreSQL: Columnar Storage - -- Suitable for small to medium amounts of historical analytics data. *(TODO: Provide more definitive guidance on what small/medium/large amounts are)* - -For better analytics performance with larger volumes of analytics data, you can continue to use the PostgreSQL adapter, but with a compatible column-based variant: - -- [cstore_fdw](https://github.com/citusdata/cstore_fdw) -- [Amazon Redshift](https://aws.amazon.com/redshift/) diff --git a/scripts/elasticsearch-v2-migrate/migrate b/scripts/elasticsearch-v2-migrate/migrate new file mode 100755 index 000000000..d05f853a5 --- /dev/null +++ b/scripts/elasticsearch-v2-migrate/migrate @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -e -u + +dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export API_UMBRELLA_RUNTIME_CONFIG="/opt/api-umbrella/var/run/runtime_config.yml" + +api-umbrella-exec resty --http-include "$dir/migrate.conf" "$dir/migrate.lua" "$@" diff --git a/scripts/elasticsearch-v2-migrate/migrate.conf b/scripts/elasticsearch-v2-migrate/migrate.conf new file mode 100644 index 000000000..b49f5c2fe --- /dev/null +++ b/scripts/elasticsearch-v2-migrate/migrate.conf @@ -0,0 +1,2 @@ +error_log stderr error; +lua_shared_dict active_config 600k; diff --git a/scripts/elasticsearch-v2-migrate/migrate.lua b/scripts/elasticsearch-v2-migrate/migrate.lua new file mode 100644 index 000000000..9f9e111e6 --- /dev/null +++ b/scripts/elasticsearch-v2-migrate/migrate.lua @@ -0,0 +1,453 @@ +local config = require "api-umbrella.proxy.models.file_config" + +-- local Date = require "pl.Date" +local argparse = require "argparse" +local elasticsearch_setup = require "api-umbrella.proxy.jobs.elasticsearch_setup" +local escape_uri_non_ascii = require "api-umbrella.utils.escape_uri_non_ascii" +local http = require "resty.http" +local inspect = require "inspect" +local json_decode = require("cjson").decode +local json_encode = require "api-umbrella.utils.json_encode" +local log_utils = require "api-umbrella.proxy.log_utils" +local luatz = require "luatz" +local nillify_json_nulls = require "api-umbrella.utils.nillify_json_nulls" +local plutils = require "pl.utils" +-- local pretty = require "pl.pretty" +local tablex = require "pl.tablex" + +-- local keys = tablex.keys +local split = plutils.split + +local bulk_size = 1000 +local args = {} + +local function table_difference(t1, t2) + local res = {} + for k,v in pairs(t1) do + if not tablex.deepcompare(t1[k], t2[k]) then res[k] = v end + end + return res +end + +local function parse_date(string) + local date + if string then + local m = ngx.re.match(string, "^(\\d{4})-(\\d{2})-(\\d{2})$") + if m then + date = luatz.timetable.new(tonumber(m[1]), tonumber(m[2]), tonumber(m[3]), 0, 0, 0) + end + end + + return date +end + +local function parse_args() + local parser = argparse("api-umbrella", "Open source API management") + + parser:option("--input", "Input Elasticsearch database URL."):count(1) + parser:option("--output", "Output Elasticsearch database URL."):count(1) + parser:option("--start-date", "Migrate data starting at this date (YYYY-MM-DD format). Defaults to earliest data available from the input database."):count("0-1") + parser:option("--end-date", "Migrate data ending on this date (YYYY-MM-DD format). Defaults to current date."):count("0-1") + parser:flag("--debug", "Debug") + + local parsed_args = parser:parse() + + local input_uri, input_err = http:parse_uri(parsed_args["input"], false) + if not input_uri then + print("--input could not be parsed. Elasticsearch URL expected.") + print(input_err) + os.exit(1) + end + + local output_uri, _ = http:parse_uri(parsed_args["output"], false) + if not output_uri then + print("--output could not be parsed. Elasticsearch URL expected.") + print(output_uri) + os.exit(1) + end + + local _, input_host, input_port = unpack(input_uri) + parsed_args["input_host"] = input_host + parsed_args["input_port"] = input_port + + local _, output_host, output_port = unpack(output_uri) + parsed_args["output_host"] = output_host + parsed_args["output_port"] = output_port + + if parsed_args["start_date"] then + parsed_args["_start_date"] = parse_date(parsed_args["start_date"]) + if not parsed_args["_start_date"] then + print("--start-date could not be parsed. YYYY-MM-DD format expected.") + os.exit(1) + end + end + + if parsed_args["end_date"] then + parsed_args["_end_date"] = parse_date(parsed_args["end_date"]) + if not parsed_args["_end_date"] then + print("--start-date could not be parsed. YYYY-MM-DD format expected.") + os.exit(1) + end + end + + return parsed_args +end + +local function elasticsearch_query(host, port, options) + local httpc = http.new() + httpc:set_timeout(120000) + httpc:connect(host, port) + local res, err = httpc:request(options) + if err then + ngx.log(ngx.ERR, "elasticsearch query failed: " .. err) + return nil, err + end + + local body, body_err = res:read_body() + if not body then + ngx.log(ngx.ERR, body_err) + return nil, body_err + end + + local keepalive_ok, keepalive_err = httpc:set_keepalive() + if not keepalive_ok then + ngx.log(ngx.ERR, keepalive_err) + end + + local response = json_decode(body) + return response +end + +local function v1_first_index_time() + local res, err = elasticsearch_query(args["input_host"], args["input_port"], { + method = "GET", + path = "/api-umbrella-logs-v1-*/_aliases", + }) + if err then + ngx.log(ngx.ERR, "unexpected error: " .. err) + return false + end + + --print(pretty.write(res)) + local months = {} + for index, _ in pairs(res) do + local m = ngx.re.match(index, "-(\\d{4})-(\\d{2})") + if m then + local date = luatz.timetable.new(tonumber(m[1]), tonumber(m[2]), 1, 0, 0, 0) + table.insert(months, date) + end + end + table.sort(months) + --print(pretty.write(months)) + return months[1] +end + +local bulk_commands = {} +local last_bulk_commands_timestamp = nil +local function flush_bulk_commands() + if #bulk_commands == 0 then + return + end + + print("\n" .. os.date("!%Y-%m-%dT%TZ") .. " - Log data from " .. os.date("!%Y-%m-%dT%TZ", last_bulk_commands_timestamp / 1000)) + + local httpc = http.new() + httpc:set_timeout(120000) + httpc:connect(config["elasticsearch"]["_first_server"]["host"], config["elasticsearch"]["_first_server"]["port"]) + + local res, err = elasticsearch_query(args["output_host"], args["output_port"], { + method = "POST", + path = "/_bulk", + headers = { + ["Content-Type"] = "application/json", + }, + body = table.concat(bulk_commands, "\n") .. "\n", + }) + if err then + ngx.log(ngx.ERR, "unexpected error: " .. err) + return false + end + + if type(res["items"]) ~= "table" then + ngx.log(ngx.ERR, "unexpected error: " .. (res["items"] or nil)) + return false + end + + local skipped_count = 0 + local created_count = 0 + local error_count = 0 + local created_ids = {} + --print(inspect(res)) + for _, item in ipairs(res["items"]) do + if item["create"]["status"] == 409 then + io.write(string.char(27) .. "[30m" .. string.char(27) .. "[2m-" .. string.char(27) .. "[0m") + skipped_count = skipped_count + 1 + elseif item["create"]["status"] == 201 then + io.write(string.char(27) .. "[32m" .. string.char(27) .. "[1m✔" .. string.char(27) .. "[0m") + created_count = created_count + 1 + table.insert(created_ids, item["create"]["_id"]) + else + io.write(string.char(27) .. "[31m" .. string.char(27) .. "[1m✖" .. string.char(27) .. "[0m") + error_count = error_count + 1 + end + end + print("") + if created_count > 0 then + print("Created: " .. created_count) + -- print("Created IDs: " .. table.concat(created_ids, ", ")) + end + if skipped_count > 0 then + print("Skipped (already exists): " .. skipped_count) + end + if error_count > 0 then + print("Errors: " .. error_count) + end + + bulk_commands = {} + last_bulk_commands_timestamp = nil +end + +local function process_hit(hit, output_index) + nillify_json_nulls(hit) + + --print(pretty.write(hit)) + local source = hit["_source"] + local data = { + api_backend_id = source["api_backend_id"], + api_backend_url_match_id = source["api_backend_url_match_id"], + legacy_api_key = source["api_key"], + denied_reason = source["gatekeeper_denied_code"], + request_accept = source["request_accept"], + request_accept_encoding = source["request_accept_encoding"], + timestamp_utc = source["request_at"], + request_basic_auth_username = source["request_basic_auth_username"], + request_connection = source["request_connection"], + request_content_type = source["request_content_type"], + request_url_hierarchy = source["request_hierarchy"], + request_url_hierarchy_level0 = source["request_url_hierarchy_level0"], + request_url_hierarchy_level1 = source["request_url_hierarchy_level1"], + request_url_hierarchy_level2 = source["request_url_hierarchy_level2"], + request_url_hierarchy_level3 = source["request_url_hierarchy_level3"], + request_url_hierarchy_level4 = source["request_url_hierarchy_level4"], + request_url_hierarchy_level5 = source["request_url_hierarchy_level5"], + request_url_hierarchy_level6 = source["request_url_hierarchy_level6"], + request_url_host = source["request_host"], + request_ip = source["request_ip"], + request_ip_city = source["request_ip_city"], + request_ip_country = source["request_ip_country"], + request_ip_region = source["request_ip_region"], + request_method = source["request_method"], + request_origin = source["request_origin"], + request_url_path = source["request_path"], + request_referer = source["request_referer"], + request_url_scheme = source["request_scheme"], + request_size = source["request_size"], + request_url_query = source["request_url_query"], + request_user_agent = source["request_user_agent"], + request_user_agent_family = source["request_user_agent_family"], + request_user_agent_type = source["request_user_agent_type"], + response_age = source["response_age"], + response_cache = source["response_cache"], + response_content_encoding = source["response_content_encoding"], + response_content_length = source["response_content_length"], + response_content_type = source["response_content_type"], + response_server = source["response_server"], + response_size = source["response_size"], + response_status = source["response_status"], + timer_response = source["response_time"], + response_transfer_encoding = source["response_transfer_encoding"], + legacy_user_email = source["user_email"], + user_id = source["user_id"], + legacy_user_registration_source = source["user_registration_source"], + } + + if type(data["timestamp_utc"]) == "string" then + data["timestamp_utc"] = luatz.parse.rfc_3339(data["timestamp_utc"]):timestamp() * 1000 + end + + log_utils.set_url_hierarchy(data) + + if not data["request_url_query"] and source["request_url"] then + local parts = split(source["request_url"], "?", true, 2) + if parts[2] then + data["request_url_query"] = escape_uri_non_ascii(parts[2]) + end + end + + local new_hit = log_utils.normalized_data(data) + local new_source = { + api_backend_id = new_hit["api_backend_id"], + api_backend_url_match_id = new_hit["api_backend_url_match_id"], + api_key = new_hit["legacy_api_key"], + gatekeeper_denied_code = new_hit["denied_reason"], + request_accept = new_hit["request_accept"], + request_accept_encoding = new_hit["request_accept_encoding"], + request_at = new_hit["timestamp_utc"], + request_basic_auth_username = new_hit["request_basic_auth_username"], + request_connection = new_hit["request_connection"], + request_content_type = new_hit["request_content_type"], + request_hierarchy = new_hit["request_url_hierarchy"], + request_url_hierarchy_level0 = new_hit["request_url_hierarchy_level0"], + request_url_hierarchy_level1 = new_hit["request_url_hierarchy_level1"], + request_url_hierarchy_level2 = new_hit["request_url_hierarchy_level2"], + request_url_hierarchy_level3 = new_hit["request_url_hierarchy_level3"], + request_url_hierarchy_level4 = new_hit["request_url_hierarchy_level4"], + request_url_hierarchy_level5 = new_hit["request_url_hierarchy_level5"], + request_url_hierarchy_level6 = new_hit["request_url_hierarchy_level6"], + request_host = new_hit["request_url_host"], + request_ip = new_hit["request_ip"], + request_ip_city = new_hit["request_ip_city"], + request_ip_country = new_hit["request_ip_country"], + request_ip_region = new_hit["request_ip_region"], + request_method = new_hit["request_method"], + request_origin = new_hit["request_origin"], + request_path = new_hit["request_url_path"], + request_referer = new_hit["request_referer"], + request_scheme = new_hit["request_url_scheme"], + request_size = new_hit["request_size"], + request_url_query = new_hit["request_url_query"], + request_user_agent = new_hit["request_user_agent"], + request_user_agent_family = new_hit["request_user_agent_family"], + request_user_agent_type = new_hit["request_user_agent_type"], + response_age = new_hit["response_age"], + response_cache = new_hit["response_cache"], + response_content_encoding = new_hit["response_content_encoding"], + response_content_length = new_hit["response_content_length"], + response_content_type = new_hit["response_content_type"], + response_server = new_hit["response_server"], + response_size = new_hit["response_size"], + response_status = new_hit["response_status"], + response_time = new_hit["timer_response"], + response_transfer_encoding = new_hit["response_transfer_encoding"], + user_email = new_hit["legacy_user_email"], + user_id = new_hit["user_id"], + user_registration_source = new_hit["legacy_user_registration_source"], + imported = source["imported"], + } + + if args["debug"] then + if #bulk_commands % 1000 == 0 then + print("DIFF - " .. inspect(table_difference(source, new_source))) + print("DIFF + " .. inspect(table_difference(new_source, source))) + end + end + + table.insert(bulk_commands, json_encode({ + create = { + _index = output_index, + _type = "log", + _id = hit["_id"], + } + })) + table.insert(bulk_commands, json_encode(new_source)) + + if not last_bulk_commands_timestamp then + last_bulk_commands_timestamp = data["timestamp_utc"] + end + + if #bulk_commands >= bulk_size * 2 then + flush_bulk_commands() + end +end + +local function search_day(date_start, date_end) + local input_index = string.format("api-umbrella-logs-v1-%04d-%02d", date_start["year"], date_start["month"]) + local output_index = string.format("api-umbrella-logs-v2-%04d-%02d-%02d", date_start["year"], date_start["month"], date_start["day"]) + local scroll_id + while true do + local res, err + if scroll_id then + res, err = elasticsearch_query(args["input_host"], args["input_port"], { + method = "GET", + path = "/_search/scroll", + query = { + scroll = "5m", + scroll_id = scroll_id, + }, + }) + else + res, err = elasticsearch_query(args["input_host"], args["input_port"], { + method = "GET", + path = "/" .. input_index .. "/_search", + query = { + scroll = "5m", + scroll_id = scroll_id, + }, + headers = { + ["Content-Type"] = "application/json", + }, + body = json_encode({ + sort = "request_at", + size = bulk_size, + query = { + range = { + request_at = { + gte = date_start:timestamp() * 1000, + lt = date_end:timestamp() * 1000, + }, + }, + }, + }) + }) + end + if err then + ngx.log(ngx.ERR, "elasticsearch query failed: " .. err) + return false + end + + scroll_id = res["_scroll_id"] + --print "." + -- print(inspect(response)) + if not res["hits"] or not res["hits"]["hits"] or #res["hits"]["hits"] == 0 then + break + end + + for _, hit in ipairs(res["hits"]["hits"]) do + process_hit(hit, output_index) + end + end + + flush_bulk_commands() + + elasticsearch_query(args["output_host"], args["output_port"], { + method = "POST", + path = "/" .. output_index .. "/_forcemerge", + query = { + max_num_segments = "1", + }, + }) +end + +local function search() + local start_date = args["_start_date"] + if not start_date then + start_date = v1_first_index_time() + end + + local end_date = args["_end_date"] + if not end_date then + end_date = luatz.now() + end + + local date = start_date + while date:timestamp() <= end_date:timestamp() do + local next_day = date:clone() + next_day["day"] = next_day["day"] + 1 + next_day:normalise() + + search_day(date, next_day) + + date = next_day + end +end + +local function run() + args = parse_args() + + elasticsearch_setup.wait_for_elasticsearch() + elasticsearch_setup.create_templates() + + search() +end + +run() diff --git a/scripts/import_access_logs/import.lua b/scripts/import_access_logs/import.lua index acb3df535..d1a937505 100644 --- a/scripts/import_access_logs/import.lua +++ b/scripts/import_access_logs/import.lua @@ -1,10 +1,9 @@ -config = require "api-umbrella.proxy.models.file_config" -require "api-umbrella.proxy.startup.init_user_agent_parser_data" +local config = require "api-umbrella.proxy.models.file_config" -inspect = require "inspect" -local cjson = require "cjson" local elasticsearch_encode_json = require "api-umbrella.utils.elasticsearch_encode_json" local http = require "resty.http" +local json_decode = require("cjson").decode +local json_encode = require "api-umbrella.utils.json_encode" local log_utils = require "api-umbrella.proxy.log_utils" local luatz = require "luatz" local user_agent_parser = require "api-umbrella.proxy.user_agent_parser" @@ -147,7 +146,7 @@ local function flush_bulk_commands() ngx.log(ngx.ERR, keepalive_err) end - local response = cjson.decode(body) + local response = json_decode(body) if type(response["items"]) ~= "table" then ngx.log(ngx.ERR, "unexpected error: " .. (body or nil)) return false @@ -319,7 +318,7 @@ local function log_request(line_matches) -- print(inspect(es_data)) local index_name = "api-umbrella-logs-v1-" .. os.date("!%Y-%m", data["timestamp_utc"] / 1000) - table.insert(bulk_commands, cjson.encode({ + table.insert(bulk_commands, json_encode({ create = { _index = index_name, _type = "log", diff --git a/scripts/rake/lint.rake b/scripts/rake/lint.rake deleted file mode 100644 index d9e85efe7..000000000 --- a/scripts/rake/lint.rake +++ /dev/null @@ -1,83 +0,0 @@ -namespace :lint do - desc "Lint JavaScript files using eslint" - task :js do - require "childprocess" - require "rainbow" - - print "Checking admin-ui... " - process = ChildProcess.build("yarn", "run", "lint:js") - process.cwd = File.join(API_UMBRELLA_SRC_ROOT, "src/api-umbrella/admin-ui") - process.io.inherit! - process.start - process.wait - exit(process.exit_code) if(process.crashed?) - puts Rainbow("OK").green.bright - end - - desc "Lint Lua files using luacheck" - task :lua do - require "childprocess" - - lua_files = `git ls-files #{API_UMBRELLA_SRC_ROOT} | grep "\.lua$"`.split("\n") - process = ChildProcess.build("build/work/test-env/vendor/bin/luacheck", *lua_files) - process.cwd = API_UMBRELLA_SRC_ROOT - process.environment["LUA_PATH"] = "build/work/test-env/vendor/share/lua/5.1/?.lua;build/work/test-env/vendor/share/lua/5.1/?/init.lua;;" - process.environment["LUA_CPATH"] = "build/work/test-env/vendor/lib/lua/5.1/?.so;;" - process.io.inherit! - process.start - process.wait - exit(process.exit_code) if(process.crashed?) - end - - require "rubocop/rake_task" - RuboCop::RakeTask.new(:ruby) do |t| - t.patterns = [ - File.join(API_UMBRELLA_SRC_ROOT, "src/api-umbrella/web-app/**/*.rb"), - File.join(API_UMBRELLA_SRC_ROOT, "test/**/*.rb"), - File.join(API_UMBRELLA_SRC_ROOT, "Rakefile"), - File.join(API_UMBRELLA_SRC_ROOT, "script/rake/*.rb"), - File.join(API_UMBRELLA_SRC_ROOT, "script/rake/*.rake"), - ] - t.options = [ - "--display-cop-names", - "--extra-details", - ] - end - - desc "Lint shell files using shellcheck" - task :shell do - require "childprocess" - require "rainbow" - - # Ignore certain vendored files for linting. - ignore_files = [ - "configure", - "templates/etc/perp/kylin/rc.run.mustache", - ] - - ["sh", "bash"].each do |shell| - shell_files = `git grep -l "^#\!/bin/#{shell}" #{API_UMBRELLA_SRC_ROOT}`.split("\n") - shell_files += `git grep -l "^#\!/usr/bin/env #{shell}" #{API_UMBRELLA_SRC_ROOT}`.split("\n") - shell_files -= ignore_files - - if(shell_files.any?) - print "Checking (#{shell}): #{shell_files.join(" ")}... " - process = ChildProcess.build("build/work/test-env/bin/shellcheck", "-s", shell, *shell_files) - process.cwd = API_UMBRELLA_SRC_ROOT - process.io.inherit! - process.start - process.wait - exit(process.exit_code) if(process.crashed?) - puts Rainbow("OK").green.bright - end - end - end -end - -desc "Lint all source code for errors and style" -task :lint do - Rake::Task["lint:lua"].invoke - Rake::Task["lint:ruby"].invoke - Rake::Task["lint:js"].invoke - Rake::Task["lint:shell"].invoke -end diff --git a/scripts/rake/maps.rake b/scripts/rake/maps.rake index bde40e70d..b3c85d837 100644 --- a/scripts/rake/maps.rake +++ b/scripts/rake/maps.rake @@ -1,4 +1,6 @@ namespace :maps do + # rubocop:disable Style/GlobalVars + task :download do [ "http://www.naturalearthdata.com/http//www.naturalearthdata.com/download/50m/cultural/ne_50m_admin_1_states_provinces_lakes.zip", @@ -89,10 +91,10 @@ namespace :maps do # individual country. countries = Oj.load(File.read(File.join($input_dir, "tmp/world-50m-combined.json"))) countries["features"].each do |feature| - File.open(File.join($output_dir, "#{feature["properties"]["iso_a2"]}.json"), "w") do |f| + File.open(File.join($output_dir, "#{feature["properties"]["iso_a2"]}.json"), "w") do |file| country = countries.dup country["features"] = [country["features"].detect { |f| f["properties"]["iso_a2"] == feature["properties"]["iso_a2"] }] - f.write(Oj.dump(country)) + file.write(Oj.dump(country)) end end end @@ -113,10 +115,10 @@ namespace :maps do feature["geometry"]["coordinates"].reject! { |c| c[0][0][0] < -177 } end - File.open(File.join($output_dir, "#{feature["properties"]["iso_3166_2"]}.json"), "w") do |f| + File.open(File.join($output_dir, "#{feature["properties"]["iso_3166_2"]}.json"), "w") do |file| state_data = data.dup state_data["features"] = [state_data["features"].detect { |f| f["properties"]["iso_3166_2"] == feature["properties"]["iso_3166_2"] }] - f.write(Oj.dump(state_data)) + file.write(Oj.dump(state_data)) end end File.open(output_path, "w") { |f| f.write(Oj.dump(data)) } @@ -180,4 +182,6 @@ namespace :maps do Rake::Task["maps:us"].invoke Rake::Task["maps:simplify"].invoke end + + # rubocop:enable Style/GlobalVars end diff --git a/scripts/rake/outdated.rake b/scripts/rake/outdated.rake index e7fa7a59a..a182f88e7 100644 --- a/scripts/rake/outdated.rake +++ b/scripts/rake/outdated.rake @@ -17,7 +17,8 @@ namespace :outdated do require "childprocess" Bundler.with_original_env do process = ChildProcess.build("bundle", "outdated") - process.cwd = API_UMBRELLA_SRC_ROOT + process.environment["BUNDLE_GEMFILE"] = File.join(API_UMBRELLA_SRC_ROOT, "Gemfile") + process.environment["BUNDLE_APP_CONFIG"] = File.join(API_UMBRELLA_SRC_ROOT, "tasks/app-deps/web-app/bundle/_persist/.bundle") process.io.inherit! process.start process.wait @@ -31,7 +32,22 @@ namespace :outdated do require "childprocess" Bundler.with_original_env do process = ChildProcess.build("bundle", "outdated") - process.cwd = File.join(API_UMBRELLA_SRC_ROOT, "src/api-umbrella/web-app") + process.environment["BUNDLE_GEMFILE"] = File.join(API_UMBRELLA_SRC_ROOT, "src/api-umbrella/web-app/Gemfile") + process.environment["BUNDLE_APP_CONFIG"] = File.join(API_UMBRELLA_SRC_ROOT, "tasks/test-deps/bundle/_persist/.bundle") + process.io.inherit! + process.start + process.wait + end + end + end + + namespace "website" do + desc "List outdated website gem dependencies" + task :gems do + require "childprocess" + Bundler.with_original_env do + process = ChildProcess.build("bundle", "outdated") + process.environment["BUNDLE_GEMFILE"] = File.join(API_UMBRELLA_SRC_ROOT, "website/Gemfile") process.io.inherit! process.start process.wait @@ -60,6 +76,10 @@ task :outdated do Rake::Task["outdated:test:gems"].invoke puts "\n\n" + puts "==== WEBSITE: GEMS ====" + Rake::Task["outdated:website:gems"].invoke + puts "\n\n" + puts "==== PACKAGES ====" Rake::Task["outdated:packages"].invoke end diff --git a/scripts/rake/outdated_packages.rb b/scripts/rake/outdated_packages.rb index dc398c7cf..3edced75a 100644 --- a/scripts/rake/outdated_packages.rb +++ b/scripts/rake/outdated_packages.rb @@ -1,6 +1,7 @@ require "json" -require "open-uri" +require "net/http" require "rainbow" +require "uri" class OutdatedPackages REPOS = { @@ -10,23 +11,24 @@ class OutdatedPackages }, "bundler" => { :git => "https://github.com/bundler/bundler.git", + # Rails 4 not compatible with 2.0 + :constraint => "~> 1.17", }, "elasticsearch" => { :git => "https://github.com/elasticsearch/elasticsearch.git", - :constraint => "~> 2.4.3", + :constraint => "~> 2.4", }, - "flume" => { - :git => "https://github.com/apache/flume.git", + "elasticsearch5" => { + :git => "https://github.com/elasticsearch/elasticsearch.git", + :constraint => "~> 5.6", + }, + "elasticsearch6" => { + :git => "https://github.com/elasticsearch/elasticsearch.git", + :constraint => "~> 6.2", }, "golang" => { :git => "https://go.googlesource.com/go", }, - "json_c" => { - :git => "https://github.com/json-c/json-c.git", - }, - "kylin" => { - :git => "https://github.com/apache/kylin.git", - }, "libcidr" => { :http => "https://www.over-yonder.net/~fullermd/projects/libcidr", }, @@ -36,67 +38,49 @@ class OutdatedPackages "libfastjson" => { :git => "https://github.com/rsyslog/libfastjson.git", }, - "libgeoip" => { - :git => "https://github.com/maxmind/geoip-api-c.git", - }, - "liblogging" => { - :git => "https://github.com/rsyslog/liblogging.git", - }, - "librdkafka" => { - :git => "https://github.com/edenhill/librdkafka.git", + "libmaxminddb" => { + :git => "https://github.com/maxmind/libmaxminddb.git", }, - "luarocks" => { - :git => "https://github.com/keplerproject/luarocks.git", - }, - "luarock_argparse" => { + "lua_argparse" => { :luarock => "argparse", }, - "luarock_cmsgpack" => { + "lua_cmsgpack" => { :luarock => "lua-cmsgpack", }, - "luarock_iconv" => { - :luarock => "lua-iconv", + "lua_icu_date" => { + :git => "https://github.com/GUI/lua-icu-date.git", + :git_ref => "master", }, - "luarock_inspect" => { + "lua_inspect" => { :luarock => "inspect", }, - "luarock_libcidr" => { - :luarock => "libcidr-ffi", + "lua_libcidr_ffi" => { + :git => "https://github.com/GUI/lua-libcidr-ffi.git", }, - "luarock_luacheck" => { + "lua_luacheck" => { :luarock => "luacheck", }, - "luarock_luaposix" => { + "lua_luaposix" => { :luarock => "luaposix", }, - "luarock_luatz" => { - :luarock => "luatz", + "lua_luasocket" => { + :git => "https://github.com/diegonehab/luasocket.git", + :git_ref => "master", }, - "luarock_lustache" => { + "lua_lustache" => { :luarock => "lustache", }, - "luarock_lyaml" => { + "lua_lyaml" => { :luarock => "lyaml", }, - "luarock_penlight" => { + "lua_penlight" => { :luarock => "penlight", }, - "luarock_resty_auto_ssl" => { + "lua_resty_auto_ssl" => { :luarock => "lua-resty-auto-ssl", }, - "luarock_resty_http" => { - :luarock => "lua-resty-http", - }, - "luarock_resty_uuid" => { - :luarock => "lua-resty-uuid", - }, - "lua_luasocket" => { - :git => "https://github.com/diegonehab/luasocket.git", - :git_ref => "master", - }, - "lua_resty_dns_cache" => { - :git => "https://github.com/hamishforbes/lua-resty-dns-cache.git", - :git_ref => "master", + "lua_resty_http" => { + :git => "https://github.com/ledgetech/lua-resty-http.git", }, "lua_resty_logger_socket" => { :git => "https://github.com/cloudflare/lua-resty-logger-socket.git", @@ -106,8 +90,14 @@ class OutdatedPackages :git => "https://github.com/cloudflare/lua-resty-shcache.git", :git_ref => "master", }, - "maven" => { - :git => "https://github.com/apache/maven.git", + "lua_resty_txid" => { + :git => "https://github.com/GUI/lua-resty-txid.git", + }, + "lua_resty_uuid" => { + :luarock => "lua-resty-uuid", + }, + "luarocks" => { + :git => "https://github.com/keplerproject/luarocks.git", }, "mailhog" => { :git => "https://github.com/mailhog/MailHog.git", @@ -123,17 +113,12 @@ class OutdatedPackages :git => "https://github.com/emicklei/mora.git", :git_ref => "master", }, - "ngx_dyups" => { - :git => "https://github.com/yzprofile/ngx_http_dyups_module.git", - :git_ref => "master", - }, - "ngx_txid" => { - :git => "https://github.com/streadway/ngx_txid.git", - :git_ref => "master", + "ngx_http_geoip2_module" => { + :git => "https://github.com/leev/ngx_http_geoip2_module.git", }, "nodejs" => { :git => "https://github.com/nodejs/node.git", - :constraint => "~> 8.10", + :constraint => "~> 10.13", }, "openldap" => { :git => "https://github.com/openldap/openldap.git", @@ -146,23 +131,19 @@ class OutdatedPackages :string_version => true, }, "pcre" => { - :http => "http://ftp.csx.cam.ac.uk/pub/software/programming/pcre/", + :http => "https://ftp.pcre.org/pub/pcre/", }, "perp" => { :http => "http://b0llix.net/perp/site.cgi?page=download", }, - "phantomjs" => { - :git => "https://github.com/ariya/phantomjs.git", - }, - "presto" => { - :git => "https://github.com/facebook/presto.git", - }, "ruby" => { :git => "https://github.com/ruby/ruby.git", :constraint => "~> 2.4.3", }, "rubygems" => { :git => "https://github.com/rubygems/rubygems.git", + # Rails 4 not compatible with Bundler 2.0 + :constraint => "~> 2.7", }, "rsyslog" => { :git => "https://github.com/rsyslog/rsyslog.git", @@ -173,20 +154,22 @@ class OutdatedPackages "shellcheck" => { :git => "https://github.com/koalaman/shellcheck.git", }, + "task" => { + :git => "https://github.com/go-task/task.git", + }, "trafficserver" => { :git => "https://github.com/apache/trafficserver.git", - :constraint => "~> 5.3.2", }, "unbound" => { - :http => "https://www.unbound.net/download.html", + :http => "https://nlnetlabs.nl/projects/unbound/download/", }, "yarn" => { :git => "https://github.com/yarnpkg/yarn.git", }, - } + }.freeze def luarocks_manifest - @luarocks_manifest ||= JSON.load(open("https://luarocks.org/manifest.json")) + @luarocks_manifest ||= JSON.parse(Net::HTTP.get_response(URI.parse("https://luarocks.org/manifest.json")).body) end def luarock_version_to_semver(version) @@ -202,7 +185,7 @@ def tag_to_semver(name, tag) # Remove prefixes containing the project name. tag.gsub!(/^#{name}[\-_]/i, "") - tag.gsub!(/^#{name.gsub("_", "-")}[\-_]/i, "") + tag.gsub!(/^#{name.tr("_", "-")}[\-_]/i, "") # Remove trailing "^{}" at end of git tags. tag.chomp!("^{}") @@ -221,24 +204,26 @@ def tag_to_semver(name, tag) tag.gsub!(/-\d{8}$/, "") when "openldap" tag.gsub!(/^rel_eng_/, "") - tag.gsub!(/_/, ".") + tag.tr!("_", ".") when "openssl", "ruby" - tag.gsub!(/_/, ".") + tag.tr!("_", ".") end tag end def initialize + seen_names = [] versions = {} - versions_content = File.read(File.join(API_UMBRELLA_SRC_ROOT, "build/cmake/versions.cmake")) + versions_content = `git grep -hE "^\\w+_version=" tasks`.strip versions_content.each_line do |line| - current_version_matches = line.match(/set\((.+?)_VERSION (.+?)\)/) + current_version_matches = line.match(/^(.+?)_version=['"]([^'"]+)/) if(!current_version_matches) next end name = current_version_matches[1].downcase + seen_names.push(name) options = REPOS[name] || {} current_version_string = current_version_matches[2] @@ -274,9 +259,9 @@ def initialize puts "#{name}: Could not parse latest commit: git ls-remote #{options[:git]} #{options[:git_ref]}" end - versions[name][:current_version] = current_commit[0,7] - versions[name][:latest_version] = latest_commit[0,7] - versions[name][:wanted_version] = latest_commit[0,7] + versions[name][:current_version] = current_commit[0, 7] + versions[name][:latest_version] = latest_commit[0, 7] + versions[name][:wanted_version] = latest_commit[0, 7] elsif(options[:git]) tags = `git ls-remote --tags #{options[:git]}`.lines tags.map! { |tag| tag_to_semver(name, tag.match(%r{refs/tags/(.+)$})[1]) } @@ -287,7 +272,7 @@ def initialize tags = luarocks_manifest["repository"][options[:luarock]].keys tags.map! { |tag| luarock_version_to_semver(tag) } elsif(options[:http]) - content = open(options[:http]).read + content = Net::HTTP.get_response(URI.parse(options[:http])).body tags = content.scan(/#{name}-[\d\.]+.tar/) tags.map! { |f| tag_to_semver(name, File.basename(f, ".tar")) } end @@ -322,7 +307,7 @@ def initialize versions[name][:wanted_version] = available_version end end - rescue ArgumentError => e + rescue ArgumentError unparsable_tags << tag end end @@ -333,6 +318,11 @@ def initialize end end + unused_repos = REPOS.keys - seen_names + if(unused_repos.any?) + puts "\n\nNOTICE: Unused repos defined in scripts/rake/outdated_packages.rb: #{unused_repos.sort.join(", ")}" + end + puts "\n\n" print Rainbow("Package".ljust(32)).underline @@ -341,7 +331,8 @@ def initialize print Rainbow("Latest".rjust(16)).underline puts "" - versions.each do |name, info| + versions.keys.sort.each do |name| + info = versions[name] name_column = name.ljust(32) if(info[:wanted_version].to_s != info[:current_version].to_s) print Rainbow(name_column).red @@ -364,4 +355,3 @@ def initialize end end end - diff --git a/src/api-umbrella/admin-ui/.bowerrc b/src/api-umbrella/admin-ui/.bowerrc deleted file mode 100644 index 959e1696e..000000000 --- a/src/api-umbrella/admin-ui/.bowerrc +++ /dev/null @@ -1,4 +0,0 @@ -{ - "directory": "bower_components", - "analytics": false -} diff --git a/src/api-umbrella/admin-ui/.editorconfig b/src/api-umbrella/admin-ui/.editorconfig index 47c543840..219985c22 100644 --- a/src/api-umbrella/admin-ui/.editorconfig +++ b/src/api-umbrella/admin-ui/.editorconfig @@ -13,22 +13,8 @@ insert_final_newline = true indent_style = space indent_size = 2 -[*.js] -indent_style = space -indent_size = 2 - [*.hbs] insert_final_newline = false -indent_style = space -indent_size = 2 - -[*.css] -indent_style = space -indent_size = 2 - -[*.html] -indent_style = space -indent_size = 2 [*.{diff,md}] trim_trailing_whitespace = false diff --git a/src/api-umbrella/admin-ui/.eslintignore b/src/api-umbrella/admin-ui/.eslintignore new file mode 100644 index 000000000..72df37307 --- /dev/null +++ b/src/api-umbrella/admin-ui/.eslintignore @@ -0,0 +1,20 @@ +# unconventional js +/blueprints/*/files/ +/vendor/ + +# compiled output +/dist/ +/tmp/ + +# dependencies +/bower_components/ +/node_modules/ + +# misc +/coverage/ +!.* + +# ember-try +/.node_modules.ember-try/ +/bower.json.ember-try +/package.json.ember-try diff --git a/src/api-umbrella/admin-ui/.eslintrc.js b/src/api-umbrella/admin-ui/.eslintrc.js index 344caed47..1fc551745 100644 --- a/src/api-umbrella/admin-ui/.eslintrc.js +++ b/src/api-umbrella/admin-ui/.eslintrc.js @@ -2,17 +2,17 @@ module.exports = { root: true, parserOptions: { ecmaVersion: 2017, - sourceType: 'module' + sourceType: 'module', }, plugins: [ - 'ember' + 'ember', ], extends: [ 'eslint:recommended', - 'plugin:ember/recommended' + 'plugin:ember/recommended', ], env: { - browser: true + browser: true, }, rules: { 'comma-dangle': ['error', 'always-multiline'], @@ -32,39 +32,28 @@ module.exports = { }, globals: { 'CommonValidations': true, - 'JsDiff': true, - '_': true, - 'ace': true, - 'bootbox': true, - 'inflection': true, - 'marked': true, }, overrides: [ // node files { files: [ - 'testem.js', + '.eslintrc.js', + '.template-lintrc.js', 'ember-cli-build.js', + 'testem.js', + 'blueprints/*/index.js', 'config/**/*.js', - 'lib/*/index.js' + 'lib/*/index.js', + 'server/**/*.js', ], parserOptions: { sourceType: 'script', - ecmaVersion: 2015 + ecmaVersion: 2015, }, env: { browser: false, - node: true - } + node: true, + }, }, - - // test files - { - files: ['tests/**/*.js'], - excludedFiles: ['tests/dummy/**/*.js'], - env: { - embertest: true - } - } - ] + ], }; diff --git a/src/api-umbrella/admin-ui/.gitignore b/src/api-umbrella/admin-ui/.gitignore index 4fcd4bf3c..c9e845464 100644 --- a/src/api-umbrella/admin-ui/.gitignore +++ b/src/api-umbrella/admin-ui/.gitignore @@ -1,18 +1,30 @@ -# See http://help.github.com/ignore-files/ for more about ignoring files. +# See https://help.github.com/ignore-files/ for more about ignoring files. # compiled output -/dist -/tmp +/dist/ +/tmp/ # dependencies -/node_modules -/bower_components +/bower_components/ +/node_modules/ # misc +/.env* +/.pnp* /.sass-cache /connect.lock -/coverage/* +/coverage/ /libpeerconnection.log -npm-debug.log -testem.log -/.eslintcache +/npm-debug.log* +/testem.log +/yarn-error.log + +# ember-try +/.node_modules.ember-try/ +/bower.json.ember-try +/package.json.ember-try + +/tmp +/node_modules +node_modules +tmp diff --git a/src/api-umbrella/admin-ui/.template-lintrc.js b/src/api-umbrella/admin-ui/.template-lintrc.js new file mode 100644 index 000000000..160caea52 --- /dev/null +++ b/src/api-umbrella/admin-ui/.template-lintrc.js @@ -0,0 +1,10 @@ +'use strict'; + +module.exports = { + extends: 'recommended', + + rules: { + 'attribute-indentation': false, + 'no-inline-styles': false, + }, +}; diff --git a/src/api-umbrella/admin-ui/.yarnrc b/src/api-umbrella/admin-ui/.yarnrc new file mode 100644 index 000000000..f49f8d056 --- /dev/null +++ b/src/api-umbrella/admin-ui/.yarnrc @@ -0,0 +1,2 @@ +--ignore-optional true +--network-timeout 300000 diff --git a/src/api-umbrella/admin-ui/README.md b/src/api-umbrella/admin-ui/README.md index b959d83da..a73a04e28 100644 --- a/src/api-umbrella/admin-ui/README.md +++ b/src/api-umbrella/admin-ui/README.md @@ -33,6 +33,12 @@ Make use of the many generators for code, try `ember help generate` for more det * `ember test` * `ember test --server` +### Linting + +* `npm run lint:hbs` +* `npm run lint:js` +* `npm run lint:js -- --fix` + ### Building * `ember build` (development) diff --git a/src/api-umbrella/admin-ui/app/adapters/application.js b/src/api-umbrella/admin-ui/app/adapters/application.js index a8db7ff64..25c57dd50 100644 --- a/src/api-umbrella/admin-ui/app/adapters/application.js +++ b/src/api-umbrella/admin-ui/app/adapters/application.js @@ -1,4 +1,8 @@ import RESTAdapter from 'ember-data/adapters/rest'; +import flatten from 'lodash-es/flatten'; +import isArray from 'lodash-es/isArray'; +import isPlainObject from 'lodash-es/isPlainObject'; +import isString from 'lodash-es/isString'; export default RESTAdapter.extend({ // Build the URL using the customizable "urlRoot" attribute that can be set @@ -33,14 +37,14 @@ export default RESTAdapter.extend({ let rawErrors = payload[key]; let normalizedErrors = []; - if(_.isArray(rawErrors)) { + if(isArray(rawErrors)) { // If an array is returned by the API, no need to process further. normalizedErrors = rawErrors; - } else if(_.isPlainObject(rawErrors)) { + } else if(isPlainObject(rawErrors)) { // Turn an object of error messages into an array of error objects. for(let field in rawErrors) { // The value might be an array of error messages. - let messages = _.flatten([rawErrors[field]]); + let messages = flatten([rawErrors[field]]); messages.forEach(function(message) { normalizedErrors.push({ field: field, @@ -48,7 +52,7 @@ export default RESTAdapter.extend({ }); }); } - } else if(_.isString(rawErrors)) { + } else if(isString(rawErrors)) { // Turn a single string error into an array. normalizedErrors = [{ message: rawErrors, diff --git a/src/api-umbrella/admin-ui/app/authenticators/devise-server-side.js b/src/api-umbrella/admin-ui/app/authenticators/devise-server-side.js index e56e0000e..e5f3ff470 100644 --- a/src/api-umbrella/admin-ui/app/authenticators/devise-server-side.js +++ b/src/api-umbrella/admin-ui/app/authenticators/devise-server-side.js @@ -1,6 +1,7 @@ import $ from 'jquery'; import Base from 'ember-simple-auth/authenticators/base'; import { Promise } from 'rsvp'; +import bootbox from 'bootbox'; import { run } from '@ember/runloop'; export default Base.extend({ diff --git a/src/api-umbrella/admin-ui/app/components/admin-groups/index-table.js b/src/api-umbrella/admin-ui/app/components/admin-groups/index-table.js index c70d9edc1..9b9fb0aa2 100644 --- a/src/api-umbrella/admin-ui/app/components/admin-groups/index-table.js +++ b/src/api-umbrella/admin-ui/app/components/admin-groups/index-table.js @@ -1,5 +1,6 @@ import Component from '@ember/component'; import DataTablesHelpers from 'api-umbrella-admin-ui/utils/data-tables-helpers'; +import escape from 'lodash-es/escape'; export default Component.extend({ didInsertElement() { @@ -13,14 +14,14 @@ export default Component.extend({ data: 'name', title: 'Name', defaultContent: '-', - render: _.bind(function(name, type, data) { + render: (name, type, data) => { if(type === 'display' && name && name !== '-') { let link = '#/admin_groups/' + data.id + '/edit'; - return '' + _.escape(name) + ''; + return '' + escape(name) + ''; } return name; - }, this), + }, }, { data: 'api_scope_display_names', diff --git a/src/api-umbrella/admin-ui/app/components/admin-groups/record-form.js b/src/api-umbrella/admin-ui/app/components/admin-groups/record-form.js index 499fd857c..62f479d41 100644 --- a/src/api-umbrella/admin-ui/app/components/admin-groups/record-form.js +++ b/src/api-umbrella/admin-ui/app/components/admin-groups/record-form.js @@ -1,20 +1,21 @@ import Component from '@ember/component'; import Save from 'api-umbrella-admin-ui/mixins/save'; +import escape from 'lodash-es/escape'; export default Component.extend(Save, { actions: { submit() { this.saveRecord({ transitionToRoute: 'admin_groups', - message: 'Successfully saved the admin group "' + _.escape(this.get('model.name')) + '"', + message: 'Successfully saved the admin group "' + escape(this.get('model.name')) + '"', }); }, delete() { this.destroyRecord({ - prompt: 'Are you sure you want to delete the admin group "' + _.escape(this.get('model.name')) + '"?', + prompt: 'Are you sure you want to delete the admin group "' + escape(this.get('model.name')) + '"?', transitionToRoute: 'admin_groups', - message: 'Successfully deleted the admin group "' + _.escape(this.get('model.name')) + '"', + message: 'Successfully deleted the admin group "' + escape(this.get('model.name')) + '"', }); }, }, diff --git a/src/api-umbrella/admin-ui/app/components/admins/index-table.js b/src/api-umbrella/admin-ui/app/components/admins/index-table.js index 46353ec8d..29d209e58 100644 --- a/src/api-umbrella/admin-ui/app/components/admins/index-table.js +++ b/src/api-umbrella/admin-ui/app/components/admins/index-table.js @@ -1,8 +1,9 @@ import $ from 'jquery'; import Component from '@ember/component'; import DataTablesHelpers from 'api-umbrella-admin-ui/utils/data-tables-helpers'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; import { computed } from '@ember/object'; +import escape from 'lodash-es/escape'; import { inject } from '@ember/service'; export default Component.extend({ @@ -20,14 +21,14 @@ export default Component.extend({ name: 'Username', title: I18n.t('mongoid.attributes.admin.username'), defaultContent: '-', - render: _.bind(function(username, type, data) { + render: (username, type, data) => { if(type === 'display' && username && username !== '-') { let link = '#/admins/' + data.id + '/edit'; - return '' + _.escape(username) + ''; + return '' + escape(username) + ''; } return username; - }, this), + }, }, { data: 'group_names', @@ -64,7 +65,7 @@ export default Component.extend({ }, downloadUrl: computed('queryParams', function() { - let params = this.get('queryParams'); + let params = this.queryParams; if(params) { params = $.param(params); } diff --git a/src/api-umbrella/admin-ui/app/components/admins/record-form.js b/src/api-umbrella/admin-ui/app/components/admins/record-form.js index 01b0c1211..7321ea554 100644 --- a/src/api-umbrella/admin-ui/app/components/admins/record-form.js +++ b/src/api-umbrella/admin-ui/app/components/admins/record-form.js @@ -1,6 +1,7 @@ import Component from '@ember/component'; import Save from 'api-umbrella-admin-ui/mixins/save'; import { computed } from '@ember/object'; +import escape from 'lodash-es/escape'; import { inject } from '@ember/service'; export default Component.extend(Save, { @@ -14,15 +15,15 @@ export default Component.extend(Save, { submit() { this.saveRecord({ transitionToRoute: 'admins', - message: 'Successfully saved the admin "' + _.escape(this.get('model.username')) + '"', + message: 'Successfully saved the admin "' + escape(this.get('model.username')) + '"', }); }, delete() { this.destroyRecord({ - prompt: 'Are you sure you want to delete the admin "' + _.escape(this.get('model.username')) + '"?', + prompt: 'Are you sure you want to delete the admin "' + escape(this.get('model.username')) + '"?', transitionToRoute: 'admins', - message: 'Successfully deleted the admin "' + _.escape(this.get('model.username')) + '"', + message: 'Successfully deleted the admin "' + escape(this.get('model.username')) + '"', }); }, }, diff --git a/src/api-umbrella/admin-ui/app/components/api-scopes/index-table.js b/src/api-umbrella/admin-ui/app/components/api-scopes/index-table.js index ad4ebe280..5cbe1d1f2 100644 --- a/src/api-umbrella/admin-ui/app/components/api-scopes/index-table.js +++ b/src/api-umbrella/admin-ui/app/components/api-scopes/index-table.js @@ -1,5 +1,6 @@ import Component from '@ember/component'; import DataTablesHelpers from 'api-umbrella-admin-ui/utils/data-tables-helpers'; +import escape from 'lodash-es/escape'; export default Component.extend({ didInsertElement() { @@ -13,14 +14,14 @@ export default Component.extend({ data: 'name', title: 'Name', defaultContent: '-', - render: _.bind(function(name, type, data) { + render: (name, type, data) => { if(type === 'display' && name && name !== '-') { let link = '#/api_scopes/' + data.id + '/edit'; - return '' + _.escape(name) + ''; + return '' + escape(name) + ''; } return name; - }, this), + }, }, { data: 'host', diff --git a/src/api-umbrella/admin-ui/app/components/api-scopes/record-form.js b/src/api-umbrella/admin-ui/app/components/api-scopes/record-form.js index c98643196..550925b15 100644 --- a/src/api-umbrella/admin-ui/app/components/api-scopes/record-form.js +++ b/src/api-umbrella/admin-ui/app/components/api-scopes/record-form.js @@ -1,20 +1,21 @@ import Component from '@ember/component'; import Save from 'api-umbrella-admin-ui/mixins/save'; +import escape from 'lodash-es/escape'; export default Component.extend(Save, { actions: { submit() { this.saveRecord({ transitionToRoute: 'api_scopes', - message: 'Successfully saved the API scope "' + _.escape(this.get('model.name')) + '"', + message: 'Successfully saved the API scope "' + escape(this.get('model.name')) + '"', }); }, delete() { this.destroyRecord({ - prompt: 'Are you sure you want to delete the API scope "' + _.escape(this.get('model.name')) + '"?', + prompt: 'Are you sure you want to delete the API scope "' + escape(this.get('model.name')) + '"?', transitionToRoute: 'api_scopes', - message: 'Successfully deleted the API scope "' + _.escape(this.get('model.name')) + '"', + message: 'Successfully deleted the API scope "' + escape(this.get('model.name')) + '"', }); }, }, diff --git a/src/api-umbrella/admin-ui/app/components/api-users/index-table.js b/src/api-umbrella/admin-ui/app/components/api-users/index-table.js index 16ba37570..2ade3b2eb 100644 --- a/src/api-umbrella/admin-ui/app/components/api-users/index-table.js +++ b/src/api-umbrella/admin-ui/app/components/api-users/index-table.js @@ -1,5 +1,6 @@ import Component from '@ember/component'; import DataTablesHelpers from 'api-umbrella-admin-ui/utils/data-tables-helpers'; +import escape from 'lodash-es/escape'; export default Component.extend({ didInsertElement() { @@ -13,14 +14,14 @@ export default Component.extend({ data: 'email', title: 'E-mail', defaultContent: '-', - render: _.bind(function(email, type, data) { + render: (email, type, data) => { if(type === 'display' && email && email !== '-') { let link = '#/api_users/' + data.id + '/edit'; - return '' + _.escape(email) + ''; + return '' + escape(email) + ''; } return email; - }, this), + }, }, { data: 'first_name', diff --git a/src/api-umbrella/admin-ui/app/components/api-users/record-form.js b/src/api-umbrella/admin-ui/app/components/api-users/record-form.js index 13ba6ff60..e45d6533a 100644 --- a/src/api-umbrella/admin-ui/app/components/api-users/record-form.js +++ b/src/api-umbrella/admin-ui/app/components/api-users/record-form.js @@ -1,6 +1,7 @@ import Component from '@ember/component'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; import Save from 'api-umbrella-admin-ui/mixins/save'; +import escape from 'lodash-es/escape'; export default Component.extend(Save, { init() { @@ -37,9 +38,9 @@ export default Component.extend(Save, { this.saveRecord({ transitionToRoute: 'api_users', message(model) { - let message = 'Successfully saved the user "' + _.escape(model.get('email')) + '"'; + let message = 'Successfully saved the user "' + escape(model.get('email')) + '"'; if(model.get('apiKey')) { - message += '
API Key: ' + _.escape(model.get('apiKey')) + ''; + message += '
API Key: ' + escape(model.get('apiKey')) + ''; } return message; diff --git a/src/api-umbrella/admin-ui/app/components/apis/index-table.js b/src/api-umbrella/admin-ui/app/components/apis/index-table.js index 15c6f05c4..7faf82e47 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/index-table.js +++ b/src/api-umbrella/admin-ui/app/components/apis/index-table.js @@ -1,7 +1,11 @@ +import 'jquery-ui/ui/widgets/sortable'; import $ from 'jquery'; import Component from '@ember/component'; import DataTablesHelpers from 'api-umbrella-admin-ui/utils/data-tables-helpers'; +import bootbox from 'bootbox'; +import escape from 'lodash-es/escape'; import { inject } from '@ember/service'; +import isEqual from 'lodash-es/isEqual'; import { observer } from '@ember/object'; export default Component.extend({ @@ -22,14 +26,14 @@ export default Component.extend({ data: 'name', title: 'Name', defaultContent: '-', - render: _.bind(function(name, type, data) { + render: (name, type, data) => { if(type === 'display' && name && name !== '-') { let link = '#/apis/' + data.id + '/edit'; - return '' + _.escape(name) + ''; + return '' + escape(name) + ''; } return name; - }, this), + }, }, { data: 'frontend_host', @@ -55,33 +59,33 @@ export default Component.extend({ className: 'reorder-handle', orderable: false, render() { - return ''; + return ''; }, }, ], })); - this.get('table') - .on('search', _.bind(function(event, settings) { + this.table + .on('search', (event, settings) => { // Disable reordering if the user tries to filter the table by anything // (otherwise, our reordering logic won't work, since it relies on the // neighboring rows). - if(this.get('reorderActive')) { + if(this.reorderActive) { if(settings.oPreviousSearch && settings.oPreviousSearch.sSearch) { this.set('reorderActive', false); } } - }, this)) - .on('order', _.bind(function(event, settings) { + }) + .on('order', (event, settings) => { // Disable reordering if the user tries to sort the table by anything // other than the sort order (otherwise, our reordering logic won't // work, since it relies on the neighboring rows). - if(this.get('reorderActive')) { - if(settings.aaSorting && !_.isEqual(settings.aaSorting, [[3, 'asc']])) { + if(this.reorderActive) { + if(settings.aaSorting && !isEqual(settings.aaSorting, [[3, 'asc']])) { this.set('reorderActive', false); } } - }, this)); + }); this.$().find('tbody').sortable({ handle: '.reorder-handle', @@ -92,7 +96,7 @@ export default Component.extend({ }); return ui; }, - stop: _.bind(function(event, ui) { + stop: (event, ui) => { let row = $(ui.item); let previousRow = row.prev('tbody tr'); let moveAfterId = null; @@ -101,14 +105,14 @@ export default Component.extend({ } this.saveReorder(row.data('id'), moveAfterId); - }, this), + }, }); }, handleReorderChange: observer('reorderActive', function() { - if(this.get('reorderActive')) { + if(this.reorderActive) { this.$().find('table').addClass('reorder-active'); - this.get('table') + this.table .order([[3, 'asc']]) .search('') .draw(); @@ -119,7 +123,7 @@ export default Component.extend({ let $container = this.$(); if($container) { let $buttonText = this.$().find('.reorder-button-text'); - if(this.get('reorderActive')) { + if(this.reorderActive) { $buttonText.data('originalText', $buttonText.text()); $buttonText.text('Done'); } else { @@ -129,25 +133,25 @@ export default Component.extend({ }), saveReorder(id, moveAfterId) { - this.get('busy').show(); + this.busy.show(); $.ajax({ url: '/api-umbrella/v1/apis/' + id + '/move_after.json', method: 'PUT', data: { move_after_id: moveAfterId }, }).done(() => { // eslint-disable-next-line ember/jquery-ember-run - this.get('table').draw(); + this.table.draw(); }).fail((xhr) => { // eslint-disable-next-line no-console console.error('Unexpected error: ' + xhr.status + ' ' + xhr.statusText + ' (' + xhr.readyState + '): ' + xhr.responseText); bootbox.alert('An unexpected error occurred. Please try again.'); - this.get('table').draw(); + this.table.draw(); }); }, actions: { toggleReorderApis() { - this.set('reorderActive', !this.get('reorderActive')); + this.set('reorderActive', !this.reorderActive); }, }, }); diff --git a/src/api-umbrella/admin-ui/app/components/apis/record-form.js b/src/api-umbrella/admin-ui/app/components/apis/record-form.js index aa058e807..24dc96b8d 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/record-form.js +++ b/src/api-umbrella/admin-ui/app/components/apis/record-form.js @@ -1,5 +1,7 @@ import Component from '@ember/component'; import Save from 'api-umbrella-admin-ui/mixins/save'; +import bootbox from 'bootbox'; +import escape from 'lodash-es/escape'; export default Component.extend(Save, { init() { @@ -21,25 +23,25 @@ export default Component.extend(Save, { submit() { this.saveRecord({ transitionToRoute: 'apis', - message: 'Successfully saved the "' + _.escape(this.get('model.name')) + '" API backend
Note: Your changes are not yet live. Publish Changes to send your updates live.', + message: 'Successfully saved the "' + escape(this.get('model.name')) + '" API backend
Note: Your changes are not yet live. Publish Changes to send your updates live.', }); }, delete() { this.destroyRecord({ - prompt: 'Are you sure you want to delete the API backend "' + _.escape(this.get('model.name')) + '"?', + prompt: 'Are you sure you want to delete the API backend "' + escape(this.get('model.name')) + '"?', transitionToRoute: 'apis', - message: 'Successfully deleted the "' + _.escape(this.get('model.name')) + '" API backend
Note: Your changes are not yet live. Publish Changes to send your updates live.', + message: 'Successfully deleted the "' + escape(this.get('model.name')) + '" API backend
Note: Your changes are not yet live. Publish Changes to send your updates live.', }); }, addUrlMatch() { - this.get('controllers.apis_url_match_form').add(this.get('model'), 'urlMatches'); + this.get('controllers.apis_url_match_form').add(this.model, 'urlMatches'); this.send('openModal', 'apis/url_match_form'); }, editUrlMatch(urlMatch) { - this.get('controllers.apis_url_match_form').edit(this.get('model'), 'urlMatches', urlMatch); + this.get('controllers.apis_url_match_form').edit(this.model, 'urlMatches', urlMatch); this.send('openModal', 'apis/url_match_form'); }, @@ -48,12 +50,12 @@ export default Component.extend(Save, { }, addSubSettings() { - this.get('controllers.apis_sub_settings_form').add(this.get('model'), 'subSettings'); + this.get('controllers.apis_sub_settings_form').add(this.model, 'subSettings'); this.send('openModal', 'apis/sub_settings_form'); }, editSubSettings(subSettings) { - this.get('controllers.apis_sub_settings_form').edit(this.get('model'), 'subSettings', subSettings); + this.get('controllers.apis_sub_settings_form').edit(this.model, 'subSettings', subSettings); this.send('openModal', 'apis/sub_settings_form'); }, @@ -62,12 +64,12 @@ export default Component.extend(Save, { }, addRewrite() { - this.get('controllers.apis_rewrite_form').add(this.get('model'), 'rewrites'); + this.get('controllers.apis_rewrite_form').add(this.model, 'rewrites'); this.send('openModal', 'apis/rewrite_form'); }, editRewrite(rewrite) { - this.get('controllers.apis_rewrite_form').edit(this.get('model'), 'rewrites', rewrite); + this.get('controllers.apis_rewrite_form').edit(this.model, 'rewrites', rewrite); this.send('openModal', 'apis/rewrite_form'); }, @@ -77,7 +79,7 @@ export default Component.extend(Save, { }, deleteChildRecord(collectionName, record, message) { - let collection = this.get('model').get(collectionName); + let collection = this.model.get(collectionName); bootbox.confirm(message, function(result) { if(result) { collection.removeObject(record); diff --git a/src/api-umbrella/admin-ui/app/components/apis/rewrite-form.js b/src/api-umbrella/admin-ui/app/components/apis/rewrite-form.js index afb7faace..95528c703 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/rewrite-form.js +++ b/src/api-umbrella/admin-ui/app/components/apis/rewrite-form.js @@ -39,21 +39,21 @@ export default Component.extend({ bufferedModel: computed('model', function() { let owner = getOwner(this).ownerInjection(); - return BufferedProxy.extend(Rewrite.validationClass).create(owner, { content: this.get('model') }); + return BufferedProxy.extend(Rewrite.validationClass).create(owner, { content: this.model }); }), actions: { submit() { - this.get('bufferedModel').applyChanges(); + this.bufferedModel.applyChanges(); if(this.get('model.isNew')) { - this.get('collection').pushObject(this.get('model')); + this.collection.pushObject(this.model); } this.set('openModal', false); }, closed() { - this.get('bufferedModel').discardChanges(); + this.bufferedModel.discardChanges(); this.set('openModal', false); }, }, diff --git a/src/api-umbrella/admin-ui/app/components/apis/rewrite-table.js b/src/api-umbrella/admin-ui/app/components/apis/rewrite-table.js index 6a1402424..1d5dba6a2 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/rewrite-table.js +++ b/src/api-umbrella/admin-ui/app/components/apis/rewrite-table.js @@ -1,5 +1,6 @@ import Component from '@ember/component'; import Sortable from 'api-umbrella-admin-ui/mixins/sortable'; +import bootbox from 'bootbox'; import { computed } from '@ember/object'; import { inject } from '@ember/service'; @@ -13,7 +14,7 @@ export default Component.extend(Sortable, { actions: { add() { - this.set('rewriteModel', this.get('store').createRecord('api/rewrite')); + this.set('rewriteModel', this.store.createRecord('api/rewrite')); this.set('openModal', true); }, diff --git a/src/api-umbrella/admin-ui/app/components/apis/server-form.js b/src/api-umbrella/admin-ui/app/components/apis/server-form.js index 88b18f739..c0d03e789 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/server-form.js +++ b/src/api-umbrella/admin-ui/app/components/apis/server-form.js @@ -17,15 +17,15 @@ export default Component.extend({ bufferedModel: computed('model', function() { let owner = getOwner(this).ownerInjection(); - return BufferedProxy.extend(Server.validationClass).create(owner, { content: this.get('model') }); + return BufferedProxy.extend(Server.validationClass).create(owner, { content: this.model }); }), actions: { open() { // For new servers, intelligently pick the default port based on the // backend protocol selected. - if(this.get('bufferedModel') && !this.get('bufferedModel.port')) { - if(this.get('apiBackendProtocol') === 'https') { + if(this.bufferedModel && !this.get('bufferedModel.port')) { + if(this.apiBackendProtocol === 'https') { this.set('bufferedModel.port', 443); } else { this.set('bufferedModel.port', 80); @@ -34,15 +34,15 @@ export default Component.extend({ }, submit() { - this.get('bufferedModel').applyChanges(); + this.bufferedModel.applyChanges(); if(this.get('model.isNew')) { - this.get('collection').pushObject(this.get('model')); + this.collection.pushObject(this.model); } // After the first server is added, fill out a default value for the // "Backend Host" field based on the server's host (because in most // non-load balancing situations they will match). - if(!this.get('apiBackendHost')) { + if(!this.apiBackendHost) { let server = this.get('collection.firstObject'); if(server && server.get('host')) { this.set('apiBackendHost', server.get('host')); @@ -53,7 +53,7 @@ export default Component.extend({ }, closed() { - this.get('bufferedModel').discardChanges(); + this.bufferedModel.discardChanges(); this.set('openModal', false); }, }, diff --git a/src/api-umbrella/admin-ui/app/components/apis/server-table.js b/src/api-umbrella/admin-ui/app/components/apis/server-table.js index f48ae94ec..80141196b 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/server-table.js +++ b/src/api-umbrella/admin-ui/app/components/apis/server-table.js @@ -1,4 +1,5 @@ import Component from '@ember/component'; +import bootbox from 'bootbox'; import { inject } from '@ember/service'; export default Component.extend({ @@ -7,7 +8,7 @@ export default Component.extend({ actions: { add() { - this.set('serverModel', this.get('store').createRecord('api/server')); + this.set('serverModel', this.store.createRecord('api/server')); this.set('openModal', true); }, diff --git a/src/api-umbrella/admin-ui/app/components/apis/settings/common-fields.js b/src/api-umbrella/admin-ui/app/components/apis/settings/common-fields.js index a221eb23f..338e2e8a8 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/settings/common-fields.js +++ b/src/api-umbrella/admin-ui/app/components/apis/settings/common-fields.js @@ -1,5 +1,5 @@ import Component from '@ember/component'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; export default Component.extend({ init() { diff --git a/src/api-umbrella/admin-ui/app/components/apis/settings/rate-limit-fields.js b/src/api-umbrella/admin-ui/app/components/apis/settings/rate-limit-fields.js index 7f0bf9912..b850852bb 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/settings/rate-limit-fields.js +++ b/src/api-umbrella/admin-ui/app/components/apis/settings/rate-limit-fields.js @@ -1,6 +1,8 @@ import Component from '@ember/component'; +import bootbox from 'bootbox'; import { computed } from '@ember/object'; import { inject } from '@ember/service'; +import uniqueId from 'lodash-es/uniqueId'; export default Component.extend({ store: inject(), @@ -30,7 +32,7 @@ export default Component.extend({ }, uniqueSettingsId: computed(function() { - return _.uniqueId('api_settings_'); + return uniqueId('api_settings_'); }), actions: { @@ -47,7 +49,7 @@ export default Component.extend({ addRateLimit() { let collection = this.get('model.rateLimits'); - collection.pushObject(this.get('store').createRecord('api/rate-limit')); + collection.pushObject(this.store.createRecord('api/rate-limit')); }, deleteRateLimit(rateLimit) { diff --git a/src/api-umbrella/admin-ui/app/components/apis/sub-settings-form.js b/src/api-umbrella/admin-ui/app/components/apis/sub-settings-form.js index 0754a68c8..5120f2ebd 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/sub-settings-form.js +++ b/src/api-umbrella/admin-ui/app/components/apis/sub-settings-form.js @@ -34,21 +34,21 @@ export default Component.extend({ bufferedModel: computed('model', function() { let owner = getOwner(this).ownerInjection(); - return BufferedProxy.extend(SubSettings.validationClass).create(owner, { content: this.get('model') }); + return BufferedProxy.extend(SubSettings.validationClass).create(owner, { content: this.model }); }), actions: { submit() { - this.get('bufferedModel').applyChanges(); + this.bufferedModel.applyChanges(); if(this.get('model.isNew')) { - this.get('collection').pushObject(this.get('model')); + this.collection.pushObject(this.model); } this.set('openModal', false); }, closed() { - this.get('bufferedModel').discardChanges(); + this.bufferedModel.discardChanges(); this.set('openModal', false); }, }, diff --git a/src/api-umbrella/admin-ui/app/components/apis/sub-settings-table.js b/src/api-umbrella/admin-ui/app/components/apis/sub-settings-table.js index 26e84be50..55879e610 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/sub-settings-table.js +++ b/src/api-umbrella/admin-ui/app/components/apis/sub-settings-table.js @@ -1,5 +1,6 @@ import Component from '@ember/component'; import Sortable from 'api-umbrella-admin-ui/mixins/sortable'; +import bootbox from 'bootbox'; import { computed } from '@ember/object'; import { inject } from '@ember/service'; @@ -13,7 +14,7 @@ export default Component.extend(Sortable, { actions: { add() { - this.set('subSettingsModel', this.get('store').createRecord('api/sub-settings')); + this.set('subSettingsModel', this.store.createRecord('api/sub-settings')); this.set('openModal', true); }, diff --git a/src/api-umbrella/admin-ui/app/components/apis/url-match-form.js b/src/api-umbrella/admin-ui/app/components/apis/url-match-form.js index 5bfc654eb..58e4adadd 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/url-match-form.js +++ b/src/api-umbrella/admin-ui/app/components/apis/url-match-form.js @@ -18,33 +18,33 @@ export default Component.extend({ bufferedModel: computed('model', function() { let owner = getOwner(this).ownerInjection(); - return BufferedProxy.extend(UrlMatch.validationClass).create(owner, { content: this.get('model') }); + return BufferedProxy.extend(UrlMatch.validationClass).create(owner, { content: this.model }); }), exampleIncomingUrl: computed('bufferedModel.frontendPrefix', function() { - let root = this.get('apiExampleIncomingUrlRoot') || ''; + let root = this.apiExampleIncomingUrlRoot || ''; let prefix = this.get('bufferedModel.frontendPrefix') || ''; - return root + prefix + this.get('exampleSuffix'); + return root + prefix + this.exampleSuffix; }), exampleOutgoingUrl: computed('bufferedModel.{frontendPrefix,backendPrefix}', function() { - let root = this.get('apiExampleOutgoingUrlRoot') || ''; + let root = this.apiExampleOutgoingUrlRoot || ''; let prefix = this.get('bufferedModel.backendPrefix') || this.get('bufferedModel.frontendPrefix') || ''; - return root + prefix + this.get('exampleSuffix'); + return root + prefix + this.exampleSuffix; }), actions: { submit() { - this.get('bufferedModel').applyChanges(); + this.bufferedModel.applyChanges(); if(this.get('model.isNew')) { - this.get('collection').pushObject(this.get('model')); + this.collection.pushObject(this.model); } this.set('openModal', false); }, closed() { - this.get('bufferedModel').discardChanges(); + this.bufferedModel.discardChanges(); this.set('openModal', false); }, }, diff --git a/src/api-umbrella/admin-ui/app/components/apis/url-match-table.js b/src/api-umbrella/admin-ui/app/components/apis/url-match-table.js index c8fbaa609..95914d036 100644 --- a/src/api-umbrella/admin-ui/app/components/apis/url-match-table.js +++ b/src/api-umbrella/admin-ui/app/components/apis/url-match-table.js @@ -1,5 +1,6 @@ import Component from '@ember/component'; import Sortable from 'api-umbrella-admin-ui/mixins/sortable'; +import bootbox from 'bootbox'; import { computed } from '@ember/object'; import { inject } from '@ember/service'; @@ -13,7 +14,7 @@ export default Component.extend(Sortable, { actions: { add() { - this.set('urlMatchModel', this.get('store').createRecord('api/url-match')); + this.set('urlMatchModel', this.store.createRecord('api/url-match')); this.set('openModal', true); }, diff --git a/src/api-umbrella/admin-ui/app/components/busy-blocker.js b/src/api-umbrella/admin-ui/app/components/busy-blocker.js new file mode 100644 index 000000000..c062d3aa1 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/components/busy-blocker.js @@ -0,0 +1,83 @@ +import Component from '@ember/component'; +import { inject } from '@ember/service'; +import { later } from '@ember/runloop'; + +const ANIMATION_DURATION = 300; +const DEFAULT_MESSAGE = 'Loading...'; + +export default Component.extend({ + busy: inject('busy'), + + classNames: ['busy-blocker'], + animationElements: null, + message: null, + + + // Hooks + // ------------------------ + didInsertElement() { + // Convert animation duration ms to css string value + let duration = (ANIMATION_DURATION / 1000) + 's'; + let busy = this.get('busy'); + + // Hide immediately + this.$().css('display', 'none'); + + this.set('animationElements', this.$('.busy-blocker__bg, .busy-blocker__content')); + // Set the animation duration on the backdrop element + this.$('.busy-blocker__bg').css('animation-duration', duration); + + busy.on('hide', this, this._hide); + busy.on('show', this, this._show); + }, + + willDestroyElement() { + let busy = this.get('busy'); + + busy.off('hide', this, this._hide); + busy.off('show', this, this._show); + }, + + + // Functions + // ------------------------ + /** + * Hide the busy animation. + * @method _hide + * @private + * @return {void} + */ + _hide() { + let elements = this.get('animationElements'); + + elements.removeClass('fade-in'); + elements.addClass('fade-out'); + + later(this, function hideLoading() { + this.$().css('display', 'none'); + }, ANIMATION_DURATION); + }, + + /** + * Show the busy animation and apply received options. + * @param {Object} [options] An optional object containing options for the busy animation + * such as a custom message. + * @method _show + * @private + * @returns {void} + */ + _show(options) { + let elements = this.get('animationElements'); + let message = DEFAULT_MESSAGE; + + if(options && options.message) { + message = options.message; + } + + this.set('message', message); + + this.$().css('display', 'block'); + elements.removeClass('fade-out'); + elements.addClass('fade-in'); + }, +}); diff --git a/src/api-umbrella/admin-ui/app/components/config/publish-form.js b/src/api-umbrella/admin-ui/app/components/config/publish-form.js index 1eb6d0392..f73b7e8db 100644 --- a/src/api-umbrella/admin-ui/app/components/config/publish-form.js +++ b/src/api-umbrella/admin-ui/app/components/config/publish-form.js @@ -1,13 +1,17 @@ import $ from 'jquery'; import Component from '@ember/component'; -import PNotify from 'npm:pnotify'; +import JsDiff from 'diff'; +import LoadingButton from 'api-umbrella-admin-ui/utils/loading-button'; +import PNotify from 'pnotify'; +import bootbox from 'bootbox'; import { computed } from '@ember/object'; +import { run } from '@ember/runloop'; export default Component.extend({ didInsertElement() { - this.$submitButton = $('#publish_button'); + this.publishButton = this.element.querySelector('.publish-button'); this.$toggleCheckboxesLink = $('#toggle_checkboxes'); - $('#publish_form').on('change', ':checkbox', _.bind(this.onCheckboxChange, this)); + $('#publish_form').on('change', ':checkbox', this.onCheckboxChange.bind(this)); let $checkboxes = $('#publish_form :checkbox'); if($checkboxes.length === 1) { @@ -57,11 +61,13 @@ export default Component.extend({ this.$toggleCheckboxesLink.text(this.$toggleCheckboxesLink.data('uncheck-all')); } - let $checked = $('#publish_form :checkbox:checked'); - if($checked.length > 0) { - this.$submitButton.prop('disabled', false); - } else { - this.$submitButton.prop('disabled', true); + if(this.publishButton) { + let $checked = $('#publish_form :checkbox:checked'); + if($checked.length > 0) { + this.publishButton.disabled = false; + } else { + this.publishButton.disabled = true; + } } }, @@ -97,24 +103,22 @@ export default Component.extend({ publish() { let form = $('#publish_form'); - let button = $('#publish_button'); - button.button('loading'); + LoadingButton.loading(this.publishButton); $.ajax({ url: '/api-umbrella/v1/config/publish', type: 'POST', data: form.serialize(), - }).then(_.bind(function() { - button.button('reset'); - new PNotify({ - type: 'success', + }).then(run.bind(this, function() { + LoadingButton.reset(this.publishButton); + PNotify.success({ title: 'Published', text: 'Successfully published the configuration
Changes should be live in a few seconds...', + textTrusted: true, }); - // eslint-disable-next-line ember/closure-actions - this.sendAction('refreshCurrentRouteController'); - }, this), function(response) { + this.get('refreshCurrentRouteController')(); + }), function(response) { let message = '

Error

'; try { let errors = response.responseJSON.errors; @@ -125,7 +129,7 @@ export default Component.extend({ message = 'An unexpected error occurred: ' + response.responseText; } - button.button('reset'); + LoadingButton.reset(this.publishButton); // eslint-disable-next-line no-console console.error(message); bootbox.alert(message); diff --git a/src/api-umbrella/admin-ui/app/components/error-messages.js b/src/api-umbrella/admin-ui/app/components/error-messages.js index 3fee4cd19..f7ec31551 100644 --- a/src/api-umbrella/admin-ui/app/components/error-messages.js +++ b/src/api-umbrella/admin-ui/app/components/error-messages.js @@ -1,6 +1,10 @@ import Component from '@ember/component'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; import { computed } from '@ember/object'; +import each from 'lodash-es/each'; +import inflection from 'inflection'; +import isArray from 'lodash-es/isArray'; +import marked from 'marked'; export default Component.extend({ messages: computed('model.{clientErrors,serverErrors}', function() { @@ -9,8 +13,8 @@ export default Component.extend({ let clientErrors = this.get('model.clientErrors'); if(clientErrors) { - if(_.isArray(clientErrors)) { - _.each(clientErrors, function(clientError) { + if(isArray(clientErrors)) { + each(clientErrors, function(clientError) { let message = clientError.get('message'); if(message) { errors.push({ @@ -28,8 +32,8 @@ export default Component.extend({ let serverErrors = this.get('model.serverErrors'); if(serverErrors) { - if(_.isArray(serverErrors)) { - _.each(serverErrors, function(serverError) { + if(isArray(serverErrors)) { + each(serverErrors, function(serverError) { let message = serverError.message; if(!message && serverError.title) { message = serverError.title; @@ -54,7 +58,7 @@ export default Component.extend({ } let messages = []; - _.each(errors, function(error) { + each(errors, function(error) { let message = ''; if(error.fullMessage) { message += error.fullMessage; @@ -81,6 +85,6 @@ export default Component.extend({ }), hasErrors: computed('messages', function() { - return (this.get('messages').length > 0); + return this.messages.length > 0; }), }); diff --git a/src/api-umbrella/admin-ui/app/components/form-fields/ace-field.js b/src/api-umbrella/admin-ui/app/components/form-fields/ace-field.js deleted file mode 100644 index c0a059659..000000000 --- a/src/api-umbrella/admin-ui/app/components/form-fields/ace-field.js +++ /dev/null @@ -1,58 +0,0 @@ -import $ from 'jquery'; -import BaseField from './base-field'; - -export default BaseField.extend({ - init() { - this._super(); - this.set('aceId', this.get('elementId') + '_ace'); - this.set('aceTextInputId', this.get('elementId') + '_ace_text_input'); - this.addObserver('model.' + this.get('fieldName'), this, this.valueDidChange); - }, - - didInsertElement() { - this._super(); - - let aceId = this.get('aceId'); - let $element = this.$().find('textarea'); - $element.hide(); - $element.before('
'); - - this.editor = ace.edit(aceId); - - let editor = this.editor; - let session = this.editor.getSession(); - - editor.$blockScrolling = Infinity; - editor.setTheme('ace/theme/textmate'); - editor.setShowPrintMargin(false); - editor.setHighlightActiveLine(false); - session.setUseWorker(false); - session.setTabSize(2); - session.setMode('ace/mode/' + $element.data('ace-mode')); - session.setValue($element.val()); - - let $textElement = $(editor.textInput.getElement()); - $textElement.attr('id', this.get('aceTextInputId')); - $textElement.attr('data-raw-input-id', $element.attr('id')); - - let contentId = this.get('elementId') + '_ace_content'; - let $content = $(editor.container).find('.ace_content'); - $content.attr('id', contentId); - $textElement.attr('data-ace-content-id', contentId); - - session.on('change', function() { - $element.val(session.getValue()); - $element.trigger('change'); - }); - }, - - valueDidChange() { - if(this.editor) { - let session = this.editor.getSession(); - let value = this.get('model.' + this.get('fieldName')); - if(value !== session.getValue()) { - session.setValue(value); - } - } - }, -}); diff --git a/src/api-umbrella/admin-ui/app/components/form-fields/base-field.js b/src/api-umbrella/admin-ui/app/components/form-fields/base-field.js index d9cdfa373..2ecccd0d9 100644 --- a/src/api-umbrella/admin-ui/app/components/form-fields/base-field.js +++ b/src/api-umbrella/admin-ui/app/components/form-fields/base-field.js @@ -3,7 +3,7 @@ import { computed } from '@ember/object'; export default Component.extend({ inputId: computed('elementId', 'fieldName', function() { - return this.get('elementId') + '-' + this.get('fieldName'); + return this.elementId + '-' + this.fieldName; }), }).reopenClass({ positionalParams: ['fieldName'], diff --git a/src/api-umbrella/admin-ui/app/components/form-fields/checkboxes-field.js b/src/api-umbrella/admin-ui/app/components/form-fields/checkboxes-field.js index 16a921a76..e821c0795 100644 --- a/src/api-umbrella/admin-ui/app/components/form-fields/checkboxes-field.js +++ b/src/api-umbrella/admin-ui/app/components/form-fields/checkboxes-field.js @@ -1,4 +1,5 @@ import BaseField from './base-field'; export default BaseField.extend({ + classNames: ['form-fields-checkboxes-field'], }); diff --git a/src/api-umbrella/admin-ui/app/components/form-fields/codemirror-field.js b/src/api-umbrella/admin-ui/app/components/form-fields/codemirror-field.js new file mode 100644 index 000000000..aea10def8 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/components/form-fields/codemirror-field.js @@ -0,0 +1,63 @@ +import 'codemirror/addon/display/autorefresh'; +import 'codemirror/mode/javascript/javascript'; +import 'codemirror/mode/xml/xml'; +import 'codemirror/mode/yaml/yaml'; +import BaseField from './base-field'; +import CodeMirror from 'codemirror/lib/codemirror' + +export default BaseField.extend({ + init() { + this._super(); + this.set('codemirrorInputFieldId', this.elementId + '_codemirror_input_field'); + this.set('codemirrorWrapperElementId', this.elementId + '_codemirror_wrapper_element'); + this.addObserver('model.' + this.fieldName, this, this.valueDidChange); + }, + + didInsertElement() { + this._super(); + + let $originalTextarea = this.$().find('textarea'); + this.codemirror = CodeMirror.fromTextArea($originalTextarea[0], { + lineNumbers: true, + mode: $originalTextarea.data('codemirror-mode'), + tabSize: 2, + + // Enable auto-refresh plugin to fix codemirror creation fields that may + // be hidden originally (eg, hidden under collapsed form sections). + autoRefresh: true, + }); + + // Set the id on the codemirror input to match the field's label so that + // when clicking on the label the codemirror input gains focus. + const inputField = this.codemirror.getInputField(); + if(inputField) { + inputField.id = this.codemirrorInputFieldId; + + const wrapperElement = this.codemirror.getWrapperElement(); + if(wrapperElement) { + wrapperElement.id = this.codemirrorWrapperElementId; + } + + inputField.setAttribute('data-codemirror-wrapper-element-id', this.codemirrorWrapperElementId); + inputField.setAttribute('data-codemirror-original-textarea-id', $originalTextarea.attr('id')); + } + + // Sync the codemirror changes back to the original textarea which will + // will update the model. + this.codemirror.on('change', () => { + this.codemirror.save(); + $originalTextarea.trigger('change'); + }); + }, + + valueDidChange() { + // Sync any external model changes back to the code mirror input. + if(this.codemirror) { + const currentValue = this.codemirror.getValue() + const newValue = this.get('model.' + this.fieldName); + if(currentValue !== newValue) { + this.codemirror.setValue(newValue); + } + } + }, +}); diff --git a/src/api-umbrella/admin-ui/app/components/form-fields/error-messages.js b/src/api-umbrella/admin-ui/app/components/form-fields/error-messages.js index bb93d73f3..8c808fe53 100644 --- a/src/api-umbrella/admin-ui/app/components/form-fields/error-messages.js +++ b/src/api-umbrella/admin-ui/app/components/form-fields/error-messages.js @@ -1,4 +1,5 @@ import Component from '@ember/component'; export default Component.extend({ + classNames: ['invalid-feedback'], }); diff --git a/src/api-umbrella/admin-ui/app/components/form-fields/field-wrapper.js b/src/api-umbrella/admin-ui/app/components/form-fields/field-wrapper.js index c5c4a7626..b06b10a1c 100644 --- a/src/api-umbrella/admin-ui/app/components/form-fields/field-wrapper.js +++ b/src/api-umbrella/admin-ui/app/components/form-fields/field-wrapper.js @@ -8,23 +8,23 @@ export default Component.extend({ canShowErrors: false, labelFor: computed('labelForId', 'inputId', function() { - return this.get('labelForId') || this.get('inputId'); + return this.labelForId || this.inputId; }), // eslint-disable-next-line ember/no-on-calls-in-components fieldNameDidChange: on('init', observer('fieldName', function() { - let fieldName = this.get('fieldName'); + let fieldName = this.fieldName; let fieldValidations = 'model.validations.attrs.' + fieldName; Ember.mixin(this, { fieldErrorMessages: computed(fieldValidations + '.messages', 'canShowErrors', function() { - if(this.get('canShowErrors')) { + if(this.canShowErrors) { return this.get(fieldValidations + '.messages'); } else { return []; } }), fieldHasErrors: computed(fieldValidations + '.isValid', 'canShowErrors', function() { - if(this.get('canShowErrors')) { + if(this.canShowErrors) { return (this.get(fieldValidations + '.isValid') === false); } else { return false; @@ -34,7 +34,7 @@ export default Component.extend({ })), wrapperErrorClass: computed('fieldHasErrors', function() { - if(this.get('fieldHasErrors')) { + if(this.fieldHasErrors) { return 'has-error'; } else { return ''; diff --git a/src/api-umbrella/admin-ui/app/components/form-fields/selectize-field.js b/src/api-umbrella/admin-ui/app/components/form-fields/selectize-field.js index 6b7d66a46..c48e2a83f 100644 --- a/src/api-umbrella/admin-ui/app/components/form-fields/selectize-field.js +++ b/src/api-umbrella/admin-ui/app/components/form-fields/selectize-field.js @@ -1,6 +1,8 @@ +import 'selectize'; import BaseField from './base-field'; import { observer } from '@ember/object'; import { on } from '@ember/object/evented'; +import uniq from 'lodash-es/uniq'; export default BaseField.extend({ optionValuePath: 'id', @@ -11,17 +13,17 @@ export default BaseField.extend({ this.defaultOptions = []; - this.set('selectizeTextInputId', this.get('elementId') + '-selectize_text_input'); - this.addObserver('model.' + this.get('fieldName'), this, this.valueDidChange); + this.set('selectizeTextInputId', this.elementId + '-selectize_text_input'); + this.addObserver('model.' + this.fieldName, this, this.valueDidChange); }, didInsertElement() { this._super(); - this.$input = this.$().find('#' + this.get('inputId')).selectize({ + this.$input = this.$().find('#' + this.inputId).selectize({ plugins: ['restore_on_backspace', 'remove_button'], delimiter: ',', - options: this.get('defaultOptions'), + options: this.defaultOptions, valueField: 'id', labelField: 'label', searchField: 'label', @@ -33,27 +35,27 @@ export default BaseField.extend({ }); this.selectize = this.$input[0].selectize; - this.selectize.$control_input.attr('id', this.get('selectizeTextInputId')); - this.selectize.$control_input.attr('data-raw-input-id', this.get('inputId')); + this.selectize.$control_input.attr('id', this.selectizeTextInputId); + this.selectize.$control_input.attr('data-raw-input-id', this.inputId); - let controlId = this.get('elementId') + '-selectize_control'; + let controlId = this.elementId + '-selectize_control'; this.selectize.$control.attr('id', controlId); this.selectize.$control_input.attr('data-selectize-control-id', controlId); }, // eslint-disable-next-line ember/no-on-calls-in-components defaultOptionsDidChange: on('init', observer('options.@each', function() { - this.set('defaultOptions', this.get('options').map(_.bind(function(item) { + this.set('defaultOptions', this.options.map((item) => { return { - id: item.get(this.get('optionValuePath')), - label: item.get(this.get('optionLabelPath')), + id: item.get(this.optionValuePath), + label: item.get(this.optionLabelPath), }; - }, this))); + })); if(this.selectize) { - this.get('defaultOptions').forEach(_.bind(function(option) { + this.defaultOptions.forEach((option) => { this.selectize.addOption(option); - }, this)); + }); this.selectize.refreshOptions(false); } @@ -63,11 +65,11 @@ export default BaseField.extend({ // externally. valueDidChange() { if(this.selectize) { - let valueString = this.get('model.' + this.get('fieldName')); + let valueString = this.get('model.' + this.fieldName); if(valueString !== this.selectize.getValue()) { let values = valueString; if(values) { - values = _.uniq(values.split(',')); + values = uniq(values.split(',')); // Ensure the selected value is available as an option in the menu. // This takes into account the fact that the default options may not diff --git a/src/api-umbrella/admin-ui/app/components/select-menu.js b/src/api-umbrella/admin-ui/app/components/select-menu.js index 6826be713..877bd5848 100644 --- a/src/api-umbrella/admin-ui/app/components/select-menu.js +++ b/src/api-umbrella/admin-ui/app/components/select-menu.js @@ -20,14 +20,13 @@ export default Component.extend({ // modals to work, since the select on those isn't set until the modal opens // (so setting a default value just on the very first render doesn't work). updateDefault() { - let value = this.get('value'); + let value = this.value; if(value === undefined) { - let options = this.get('options'); + let options = this.options; if(options) { let firstOption = options[0]; if(firstOption && firstOption.id) { - // eslint-disable-next-line ember/closure-actions - this.sendAction('action', firstOption.id, this); + this.get('action')(firstOption.id); } } } diff --git a/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-breadcrumbs.js b/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-breadcrumbs.js index 854fdc55b..accdbea9b 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-breadcrumbs.js +++ b/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-breadcrumbs.js @@ -5,7 +5,7 @@ export default Component.extend({ breadcrumbLinks: computed('breadcrumbs', function() { let crumbs = []; - let data = this.get('breadcrumbs'); + let data = this.breadcrumbs; for(let i = 0; i < data.length; i++) { let crumb = { name: data[i].crumb }; if(i < data.length - 1) { diff --git a/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-chart.js b/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-chart.js index ec3a4bdf7..a89bf48a3 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-chart.js +++ b/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-chart.js @@ -1,6 +1,7 @@ import $ from 'jquery'; import Component from '@ember/component'; -import echarts from 'npm:echarts'; +import debounce from 'lodash-es/debounce'; +import echarts from 'echarts/lib/echarts'; import { observer } from '@ember/object'; import { on } from '@ember/object/evented'; @@ -15,7 +16,7 @@ export default Component.extend({ this.chart = echarts.init(this.$()[0], 'api-umbrella-theme'); this.draw(); - $(window).on('resize', _.debounce(this.chart.resize, 100)); + $(window).on('resize', debounce(this.chart.resize, 100)); }, // eslint-disable-next-line ember/no-on-calls-in-components @@ -23,7 +24,7 @@ export default Component.extend({ let data = [] let labels = []; - let hits = this.get('hitsOverTime'); + let hits = this.hitsOverTime; for(let i = 1; i < hits.cols.length; i++) { data.push({ name: hits.cols[i].label, @@ -59,7 +60,7 @@ export default Component.extend({ })), draw() { - if(!this.chart || !this.get('chartData')) { + if(!this.chart || !this.chartData) { return; } @@ -101,9 +102,9 @@ export default Component.extend({ xAxis: { type: 'category', boundaryGap: false, - data: this.get('chartLabels'), + data: this.chartLabels, }, - series: this.get('chartData'), + series: this.chartData, title: { show: false, }, diff --git a/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-table.js b/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-table.js index 372844e3d..50e5040f9 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-table.js +++ b/src/api-umbrella/admin-ui/app/components/stats/drilldown/results-table.js @@ -2,6 +2,8 @@ import { computed, observer } from '@ember/object'; import $ from 'jquery'; import Component from '@ember/component'; +import clone from 'lodash-es/clone'; +import escape from 'lodash-es/escape'; import { inject } from '@ember/service'; import numeral from 'numeral'; @@ -12,7 +14,7 @@ export default Component.extend({ this.$().find('table').DataTable({ searching: false, order: [[1, 'desc']], - data: this.get('results'), + data: this.results, columns: [ { data: 'path', @@ -21,13 +23,13 @@ export default Component.extend({ render: function(name, type, data) { if(type === 'display' && name && name !== '-') { if(data.terminal) { - return '' + _.escape(name); + return '' + escape(name); } else { - let params = _.clone(this.get('presentQueryParamValues')); + let params = clone(this.presentQueryParamValues); params.prefix = data.descendent_prefix; let link = '#/stats/drilldown?' + $.param(params); - return '' + _.escape(name) + ''; + return '' + escape(name) + ''; } } @@ -53,11 +55,11 @@ export default Component.extend({ refreshData: observer('results', function() { let table = this.$().find('table').dataTable().api(); table.clear(); - table.rows.add(this.get('results')); + table.rows.add(this.results); table.draw(); }), downloadUrl: computed('backendQueryParamValues', function() { - return '/api-umbrella/v1/analytics/drilldown.csv?api_key=' + this.get('session.data.authenticated.api_key') + '&' + $.param(this.get('backendQueryParamValues')); + return '/api-umbrella/v1/analytics/drilldown.csv?api_key=' + this.get('session.data.authenticated.api_key') + '&' + $.param(this.backendQueryParamValues); }), }); diff --git a/src/api-umbrella/admin-ui/app/components/stats/logs/results-chart.js b/src/api-umbrella/admin-ui/app/components/stats/logs/results-chart.js index f5cdefe55..4b26a6c39 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/logs/results-chart.js +++ b/src/api-umbrella/admin-ui/app/components/stats/logs/results-chart.js @@ -1,6 +1,7 @@ import $ from 'jquery'; import Component from '@ember/component'; -import echarts from 'npm:echarts'; +import debounce from 'lodash-es/debounce'; +import echarts from 'echarts/lib/echarts'; import { observer } from '@ember/object'; import { on } from '@ember/object/evented'; @@ -15,7 +16,7 @@ export default Component.extend({ this.chart = echarts.init(this.$()[0], 'api-umbrella-theme'); this.draw(); - $(window).on('resize', _.debounce(this.chart.resize, 100)); + $(window).on('resize', debounce(this.chart.resize, 100)); }, // eslint-disable-next-line ember/no-on-calls-in-components @@ -23,7 +24,7 @@ export default Component.extend({ let data = [] let labels = []; - let hits = this.get('hitsOverTime'); + let hits = this.hitsOverTime; for(let i = 0; i < hits.length; i++) { data.push(hits[i].c[1].v); labels.push(hits[i].c[0].f); @@ -38,13 +39,13 @@ export default Component.extend({ })), draw() { - if(!this.chart || !this.get('chartData')) { + if(!this.chart || !this.chartData) { return; } let showAllSymbol = false; let lineWidth = 2; - if(this.get('chartData').length < 100) { + if(this.chartData.length < 100) { showAllSymbol = true; lineWidth = 4; } @@ -87,7 +88,7 @@ export default Component.extend({ xAxis: { type: 'category', boundaryGap: false, - data: this.get('chartLabels'), + data: this.chartLabels, }, series: [ { @@ -104,7 +105,7 @@ export default Component.extend({ width: lineWidth, }, }, - data: this.get('chartData'), + data: this.chartData, }, ], title: { diff --git a/src/api-umbrella/admin-ui/app/components/stats/logs/results-facet-table.js b/src/api-umbrella/admin-ui/app/components/stats/logs/results-facet-table.js index 02f7fd526..a03bfab2a 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/logs/results-facet-table.js +++ b/src/api-umbrella/admin-ui/app/components/stats/logs/results-facet-table.js @@ -1,14 +1,17 @@ import $ from 'jquery'; import Component from '@ember/component'; +import clone from 'lodash-es/clone'; +import compact from 'lodash-es/compact'; +import each from 'lodash-es/each'; import { observer } from '@ember/object'; import { on } from '@ember/object/evented'; export default Component.extend({ // eslint-disable-next-line ember/no-on-calls-in-components setLinks: on('init', observer('facets', function() { - _.each(this.get('facets'), function(bucket) { - let params = _.clone(this.get('presentQueryParamValues')); - params.search = _.compact([params.search, this.get('field') + ':"' + bucket.key + '"']).join(' AND '); + each(this.facets, function(bucket) { + let params = clone(this.presentQueryParamValues); + params.search = compact([params.search, this.field + ':"' + bucket.key + '"']).join(' AND '); bucket.link = '#/stats/logs?' + $.param(params); }.bind(this)); })), diff --git a/src/api-umbrella/admin-ui/app/components/stats/logs/results-table.js b/src/api-umbrella/admin-ui/app/components/stats/logs/results-table.js index 8c64e4970..b1aecfa4b 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/logs/results-table.js +++ b/src/api-umbrella/admin-ui/app/components/stats/logs/results-table.js @@ -3,6 +3,11 @@ import { computed, observer } from '@ember/object'; import $ from 'jquery'; import Component from '@ember/component'; import DataTablesHelpers from 'api-umbrella-admin-ui/utils/data-tables-helpers'; +import clone from 'lodash-es/clone'; +import compact from 'lodash-es/compact'; +import escape from 'lodash-es/escape'; +import extend from 'lodash-es/extend'; +import tippy from 'tippy.js' export default Component.extend({ didInsertElement() { @@ -15,32 +20,24 @@ export default Component.extend({ // exceed URL length limits in IE (and apparently Capybara too). type: 'POST', data: function(data) { - return _.extend({}, data, this.get('backendQueryParamValues')); + return extend({}, data, this.backendQueryParamValues); }.bind(this), }, - drawCallback: _.bind(function() { + drawCallback: () => { this.$().find('td').each(function() { if(this.scrollWidth > this.offsetWidth) { const $cell = $(this); - $cell.prop('title', $cell.text()); + $cell.attr('data-tippy-content', $cell.text()); - $cell.qtip({ - style: { - classes: 'qtip-bootstrap qtip-forced-wide', - }, - hide: { - fixed: true, - delay: 200, - }, - position: { - viewport: false, - my: 'bottom center', - at: 'top center', - }, + tippy($cell[0], { + interactive: true, + theme: 'light-border forced-wide', + arrow: true, + delay: 200, }); } }); - }, this), + }, order: [[0, 'desc']], columns: [ { @@ -74,11 +71,11 @@ export default Component.extend({ defaultContent: '-', render: function(email, type, data) { if(type === 'display' && email && email !== '-') { - let params = _.clone(this.get('presentQueryParamValues')); - params.search = _.compact([params.search, 'user_id:"' + data.user_id + '"']).join(' AND '); + let params = clone(this.presentQueryParamValues); + params.search = compact([params.search, 'user_id:"' + data.user_id + '"']).join(' AND '); let link = '#/stats/logs?' + $.param(params); - return '' + _.escape(email) + ''; + return '' + escape(email) + ''; } return email; @@ -183,6 +180,6 @@ export default Component.extend({ }), downloadUrl: computed('backendQueryParamValues', function() { - return '/admin/stats/logs.csv?' + $.param(this.get('backendQueryParamValues')); + return '/admin/stats/logs.csv?' + $.param(this.backendQueryParamValues); }), }); diff --git a/src/api-umbrella/admin-ui/app/components/stats/map/results-breadcrumbs.js b/src/api-umbrella/admin-ui/app/components/stats/map/results-breadcrumbs.js index e054b5c04..4e8f831ee 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/map/results-breadcrumbs.js +++ b/src/api-umbrella/admin-ui/app/components/stats/map/results-breadcrumbs.js @@ -5,7 +5,7 @@ export default Component.extend({ breadcrumbLinks: computed('breadcrumbs', function() { let crumbs = []; - let data = this.get('breadcrumbs'); + let data = this.breadcrumbs; for(let i = 0; i < data.length; i++) { let crumb = { name: data[i].name }; if(i < data.length - 1) { diff --git a/src/api-umbrella/admin-ui/app/components/stats/map/results-map.js b/src/api-umbrella/admin-ui/app/components/stats/map/results-map.js index 533d5c396..598afa0fb 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/map/results-map.js +++ b/src/api-umbrella/admin-ui/app/components/stats/map/results-map.js @@ -1,6 +1,8 @@ import $ from 'jquery'; import Component from '@ember/component'; -import echarts from 'npm:echarts'; +import clone from 'lodash-es/clone'; +import debounce from 'lodash-es/debounce'; +import echarts from 'echarts/lib/echarts'; import { inject } from '@ember/service'; import { observer } from '@ember/object'; import { on } from '@ember/object/evented'; @@ -20,13 +22,13 @@ export default Component.extend({ this.chart.on('click', this.handleCityClick.bind(this)); this.draw(); - $(window).on('resize', _.debounce(this.chart.resize, 100)); + $(window).on('resize', debounce(this.chart.resize, 100)); }, handleRegionClick(event) { - let queryParams = _.clone(this.get('presentQueryParamValues')); + let queryParams = clone(this.presentQueryParamValues); queryParams.region = event.batch[0].name; - this.get('router').transitionTo('stats.map', { queryParams }); + this.router.transitionTo('stats.map', { queryParams }); }, handleCityClick(event) { @@ -34,7 +36,7 @@ export default Component.extend({ let currentRegion = this.get('allQueryParamValues.region').split('-'); let currentCountry = currentRegion[0]; currentRegion = currentRegion[1]; - let queryParams = _.clone(this.get('presentQueryParamValues')); + let queryParams = clone(this.presentQueryParamValues); queryParams.query = JSON.stringify({ condition: 'AND', rules: [ @@ -73,7 +75,7 @@ export default Component.extend({ ], }); - this.get('router').transitionTo('stats.logs', { queryParams }); + this.router.transitionTo('stats.logs', { queryParams }); } }, @@ -102,6 +104,8 @@ export default Component.extend({ echarts.registerMap('region', geojson, specialMapAreas); this.set('loadedMapRegion', this.get('allQueryParamValues.region')); + + this.fillInChartDataMissingRegions(); this.draw(); }); })), @@ -110,11 +114,11 @@ export default Component.extend({ refreshData: on('init', observer('regions', function() { let currentRegion = this.get('allQueryParamValues.region'); - let data = []; + let data = {}; let maxValue = 2; let maxValueDisplay = '2'; - let hits = this.get('regions'); - let regionField = this.get('regionField'); + let hits = this.regions; + let regionField = this.regionField; for(let i = 0; i < hits.length; i++) { let value, valueDisplay; if(regionField === 'request_ip_city') { @@ -122,11 +126,11 @@ export default Component.extend({ valueDisplay = hits[i].c[3].f; let lat = hits[i].c[0].v; let lng = hits[i].c[1].v; - data.push({ + data[i] = { name: hits[i].c[2].v, value: [lng, lat, value], valueDisplay: valueDisplay, - }); + }; } else { value = hits[i].c[1].v; valueDisplay = hits[i].c[1].f; @@ -135,11 +139,11 @@ export default Component.extend({ code = 'US-' + code; } - data.push({ + data[code] = { name: code, value: value, valueDisplay: valueDisplay, - }); + }; } if(value > maxValue) { @@ -153,30 +157,55 @@ export default Component.extend({ this.set('chartDataMaxValueDisplay', maxValueDisplay); this.set('loadedDataRegion', this.get('allQueryParamValues.region')); + this.fillInChartDataMissingRegions(); this.draw(); })), + // In order to generate tooltips with the region names, the region data must + // contain a record for each region, even if no data is present (otherwise + // the "params" passed to the tooltip's formatter function doesn't contain + // the hovered region code as of ECharts 4). To fix this when no data is + // present, ensure that anytime the chart data or labels are changed, this + // function gets called to fill in any missing data. + fillInChartDataMissingRegions() { + if(this.chartData && this.labels && this.regionField !== 'request_ip_city') { + let data = this.chartData + const regionCodes = Object.keys(this.labels); + for(let i = 0, len = regionCodes.length; i < len; i++) { + const regionCode = regionCodes[i]; + if(!data[regionCode]) { + data[regionCode] = { + name: regionCode, + }; + } + } + + this.set('chartData', data); + } + }, + draw() { let currentRegion = this.get('allQueryParamValues.region'); - if(!this.chart || this.get('loadedDataRegion') !== currentRegion || this.get('loadedMapRegion') !== currentRegion) { + if(!this.chart || this.loadedDataRegion !== currentRegion || this.loadedMapRegion !== currentRegion) { return; } let geo; let series = {}; - if(this.get('regionField') === 'request_ip_city') { + const data = Object.values(this.chartData); + if(this.regionField === 'request_ip_city') { geo = { map: 'region', silent: true, }; - let maxValue = this.get('chartDataMaxValue'); + let maxValue = this.chartDataMaxValue; series = [ { name: 'Hits Scatter', type: 'scatter', coordinateSystem: 'geo', - data: this.get('chartData'), + data, symbolSize: (val) => { return Math.max(Math.round((val[2] / maxValue) * 30), 6); }, @@ -189,7 +218,7 @@ export default Component.extend({ type: 'map', map: 'region', selectedMode: 'single', - data: this.get('chartData'), + data, }, ]; } @@ -201,7 +230,7 @@ export default Component.extend({ trigger: 'item', formatter: function(params) { let label = this.labels[params.name] || params.name; - let valueDisplay = params.data.valueDisplay || 0; + let valueDisplay = (params.data && params.data.valueDisplay) ? params.data.valueDisplay : 0; return '' + label + '
Hits: ' + valueDisplay + ''; }.bind(this), }, @@ -225,10 +254,10 @@ export default Component.extend({ visualMap: { type: 'continuous', min: 1, - max: this.get('chartDataMaxValue'), + max: this.chartDataMaxValue, orient: 'horizontal', text: [ - this.get('chartDataMaxValueDisplay'), + this.chartDataMaxValueDisplay, '1', ], }, diff --git a/src/api-umbrella/admin-ui/app/components/stats/map/results-table.js b/src/api-umbrella/admin-ui/app/components/stats/map/results-table.js index b80b5e909..0fbda69c6 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/map/results-table.js +++ b/src/api-umbrella/admin-ui/app/components/stats/map/results-table.js @@ -2,6 +2,8 @@ import { computed, observer } from '@ember/object'; import $ from 'jquery'; import Component from '@ember/component'; +import clone from 'lodash-es/clone'; +import escape from 'lodash-es/escape'; import numeral from 'numeral'; export default Component.extend({ @@ -9,17 +11,17 @@ export default Component.extend({ this.$().find('table').DataTable({ searching: false, order: [[1, 'desc']], - data: this.get('regions'), + data: this.regions, columns: [ { data: 'name', title: 'Location', defaultContent: '-', - render: _.bind(function(name, type, data) { + render: (name, type, data) => { if(type === 'display' && name && name !== '-') { let link; - let params = _.clone(this.get('presentQueryParamValues')); - if(this.get('regionField') === 'request_ip_city') { + let params = clone(this.presentQueryParamValues); + if(this.regionField === 'request_ip_city') { delete params.region; params.search = 'request_ip_city:"' + data.id + '"'; link = '#/stats/logs?' + $.param(params); @@ -28,11 +30,11 @@ export default Component.extend({ link = '#/stats/map?' + $.param(params); } - return '' + _.escape(name) + ''; + return '' + escape(name) + ''; } return name; - }, this), + }, }, { data: 'hits', @@ -53,11 +55,11 @@ export default Component.extend({ refreshData: observer('regions', function() { let table = this.$().find('table').dataTable().api(); table.clear(); - table.rows.add(this.get('regions')); + table.rows.add(this.regions); table.draw(); }), downloadUrl: computed('backendQueryParamValues', function() { - return '/admin/stats/map.csv?' + $.param(this.get('backendQueryParamValues')); + return '/admin/stats/map.csv?' + $.param(this.backendQueryParamValues); }), }); diff --git a/src/api-umbrella/admin-ui/app/components/stats/query-form.js b/src/api-umbrella/admin-ui/app/components/stats/query-form.js index 18af0b529..6b32e4fa1 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/query-form.js +++ b/src/api-umbrella/admin-ui/app/components/stats/query-form.js @@ -1,12 +1,34 @@ -import 'npm:bootstrap-daterangepicker'; +import 'daterangepicker'; import $ from 'jquery'; import Component from '@ember/component'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; +import QueryBuilder from 'jQuery-QueryBuilder'; +import forEach from 'lodash-es/forEach'; import { inject } from '@ember/service'; -import moment from 'npm:moment-timezone'; +import moment from 'moment-timezone'; import { observer } from '@ember/object'; +QueryBuilder.define('filter-description', function() { + this.on('afterUpdateRuleFilter afterUpdateRuleOperator', function(e, rule) { + let $b = rule.$el.find('button.filter-description'); + const description = e.builder.getFilterDescription(rule.filter, rule); + + if(!description) { + $b.hide(); + } else { + if($b.length === 0) { + $b = $(''); + $b.prependTo(rule.$el.find(QueryBuilder.selectors.rule_actions)); + } else { + $b.css('display', ''); + } + + $b.attr('data-tippy-content', description); + } + }); +}); + export default Component.extend({ session: inject('session'), @@ -15,7 +37,7 @@ export default Component.extend({ didInsertElement() { let rangeOptions = {}; let rangeKeys = {}; - _.forEach(this.get('dateRanges'), function(range, key) { + forEach(this.dateRanges, function(range, key) { rangeOptions[range.label] = [ range.start_at, range.end_at, @@ -28,6 +50,9 @@ export default Component.extend({ let $dateRangePicker = $('#reportrange'); $dateRangePicker.daterangepicker({ ranges: rangeOptions, + showDropdowns: true, + minYear: 2000, + maxYear: new Date().getFullYear() + 1, }); $dateRangePicker.on('showCalendar.daterangepicker', this.handleDateRangeCalendarShow.bind(this)); $dateRangePicker.on('hideCalendar.daterangepicker', this.handleDateRangeCalendarHide.bind(this)); @@ -69,10 +94,8 @@ export default Component.extend({ let $queryBuilder = $('#query_builder').queryBuilder({ plugins: { 'filter-description': { - icon: 'fa fa-info-circle', - mode: 'bootbox', + mode: 'tippy', }, - 'bt-tooltip-errors': null, }, allow_empty: true, allow_groups: false, @@ -254,7 +277,7 @@ export default Component.extend({ ], }); - let query = this.get('query'); + let query = this.query; let rules; if(query) { rules = JSON.parse(query); @@ -265,16 +288,14 @@ export default Component.extend({ $queryBuilder.queryBuilder('setRules', rules); } - this.send('toggleFilters'); this.send('toggleFilterType', 'builder'); - } else if(this.get('search')) { - this.send('toggleFilters'); + } else if(this.search) { this.send('toggleFilterType', 'advanced'); } }, updateQueryBuilderRules: observer('query', function() { - let query = this.get('query'); + let query = this.query; let rules; if(query) { rules = JSON.parse(query); @@ -321,9 +342,9 @@ export default Component.extend({ // predefined range. To workaround this issue (so any dates picked when // "Custom Range" is open are treated the same), we check to see if the // "Custom Range" calendars are visible or not. - let rangeOptions = this.get('rangeOptions'); - if(rangeOptions[picker.chosenLabel] && !this.get('calendarShown')) { - let rangeKeys = this.get('rangeKeys'); + let rangeOptions = this.rangeOptions; + if(rangeOptions[picker.chosenLabel] && !this.calendarShown) { + let rangeKeys = this.rangeKeys; this.setProperties({ start_at: '', end_at: '', @@ -343,20 +364,6 @@ export default Component.extend({ }, actions: { - toggleFilters() { - let $container = $('#filters_ui'); - let $icon = $('#filter_toggle .fa'); - if($container.is(':visible')) { - $icon.addClass('fa-caret-right'); - $icon.removeClass('fa-caret-down'); - } else { - $icon.addClass('fa-caret-down'); - $icon.removeClass('fa-caret-right'); - } - - $container.slideToggle(100); - }, - toggleFilterType(type) { $('.filter-type').hide(); $('#filter_type_' + type).show(); diff --git a/src/api-umbrella/admin-ui/app/components/stats/users/results-table.js b/src/api-umbrella/admin-ui/app/components/stats/users/results-table.js index 4ceefb274..2f0080745 100644 --- a/src/api-umbrella/admin-ui/app/components/stats/users/results-table.js +++ b/src/api-umbrella/admin-ui/app/components/stats/users/results-table.js @@ -3,6 +3,9 @@ import { computed, observer } from '@ember/object'; import $ from 'jquery'; import Component from '@ember/component'; import DataTablesHelpers from 'api-umbrella-admin-ui/utils/data-tables-helpers'; +import clone from 'lodash-es/clone'; +import escape from 'lodash-es/escape'; +import extend from 'lodash-es/extend'; import numeral from 'numeral'; export default Component.extend({ @@ -13,7 +16,7 @@ export default Component.extend({ ajax: { url: '/admin/stats/users.json', data: function(data) { - return _.extend({}, data, this.get('backendQueryParamValues')); + return extend({}, data, this.backendQueryParamValues); }.bind(this), }, order: [[4, 'desc']], @@ -24,11 +27,11 @@ export default Component.extend({ defaultContent: '-', render: function(email, type, data) { if(type === 'display' && email && email !== '-') { - let params = _.clone(this.get('presentQueryParamValues')); + let params = clone(this.presentQueryParamValues); params.search = 'user_id:"' + data.id + '"'; let link = '#/stats/logs?' + $.param(params); - return '' + _.escape(email) + ''; + return '' + escape(email) + ''; } return email; @@ -87,6 +90,6 @@ export default Component.extend({ }), downloadUrl: computed('backendQueryParamValues', function() { - return '/admin/stats/users.csv?' + $.param(this.get('backendQueryParamValues')); + return '/admin/stats/users.csv?' + $.param(this.backendQueryParamValues); }), }); diff --git a/src/api-umbrella/admin-ui/app/components/website-backends/index-table.js b/src/api-umbrella/admin-ui/app/components/website-backends/index-table.js index 94adeab07..261231574 100644 --- a/src/api-umbrella/admin-ui/app/components/website-backends/index-table.js +++ b/src/api-umbrella/admin-ui/app/components/website-backends/index-table.js @@ -1,5 +1,6 @@ import $ from 'jquery'; import Component from '@ember/component'; +import escape from 'lodash-es/escape'; export default Component.extend({ didInsertElement() { @@ -16,14 +17,14 @@ export default Component.extend({ data: 'frontend_host', title: 'Host', defaultContent: '-', - render: _.bind(function(name, type, data) { + render: (name, type, data) => { if(type === 'display' && name && name !== '-') { let link = '#/website_backends/' + data.id + '/edit'; - return '' + _.escape(name) + ''; + return '' + escape(name) + ''; } return name; - }, this), + }, }, ], })); diff --git a/src/api-umbrella/admin-ui/app/components/website-backends/record-form.js b/src/api-umbrella/admin-ui/app/components/website-backends/record-form.js index 5ad5553b9..e6a325bc9 100644 --- a/src/api-umbrella/admin-ui/app/components/website-backends/record-form.js +++ b/src/api-umbrella/admin-ui/app/components/website-backends/record-form.js @@ -1,5 +1,6 @@ import Component from '@ember/component'; import Save from 'api-umbrella-admin-ui/mixins/save'; +import escape from 'lodash-es/escape'; import { observer } from '@ember/object'; export default Component.extend(Save, { @@ -30,15 +31,15 @@ export default Component.extend(Save, { submit() { this.saveRecord({ transitionToRoute: 'website_backends', - message: 'Successfully saved the "' + _.escape(this.get('model.frontendHost')) + '" website backend
Note: Your changes are not yet live. Publish Changes to send your updates live.', + message: 'Successfully saved the "' + escape(this.get('model.frontendHost')) + '" website backend
Note: Your changes are not yet live. Publish Changes to send your updates live.', }); }, delete() { this.destroyRecord({ - prompt: 'Are you sure you want to delete the website backend "' + _.escape(this.get('model.frontendHost')) + '"?', + prompt: 'Are you sure you want to delete the website backend "' + escape(this.get('model.frontendHost')) + '"?', transitionToRoute: 'website_backends', - message: 'Successfully deleted the "' + _.escape(this.get('model.frontendHost')) + '" website backend
Note: Your changes are not yet live. Publish Changes to send your updates live.', + message: 'Successfully deleted the "' + escape(this.get('model.frontendHost')) + '" website backend
Note: Your changes are not yet live. Publish Changes to send your updates live.', }); }, }, diff --git a/src/api-umbrella/admin-ui/app/controllers/application.js b/src/api-umbrella/admin-ui/app/controllers/application.js index 77502b91b..5dc8bafda 100644 --- a/src/api-umbrella/admin-ui/app/controllers/application.js +++ b/src/api-umbrella/admin-ui/app/controllers/application.js @@ -13,7 +13,7 @@ export default Controller.extend({ actions: { logout() { - this.get('session').invalidate(); + this.session.invalidate(); }, }, }); diff --git a/src/api-umbrella/admin-ui/app/controllers/stats/base.js b/src/api-umbrella/admin-ui/app/controllers/stats/base.js index 7aeac1063..746c64911 100644 --- a/src/api-umbrella/admin-ui/app/controllers/stats/base.js +++ b/src/api-umbrella/admin-ui/app/controllers/stats/base.js @@ -19,5 +19,4 @@ export default Controller.extend({ value: null, }], }), - beta_analytics: false, }); diff --git a/src/api-umbrella/admin-ui/app/controllers/stats/drilldown.js b/src/api-umbrella/admin-ui/app/controllers/stats/drilldown.js index 75fb56636..b6d1221ce 100644 --- a/src/api-umbrella/admin-ui/app/controllers/stats/drilldown.js +++ b/src/api-umbrella/admin-ui/app/controllers/stats/drilldown.js @@ -8,6 +8,5 @@ export default Base.extend({ 'query', 'search', 'prefix', - 'beta_analytics', ], }); diff --git a/src/api-umbrella/admin-ui/app/controllers/stats/logs.js b/src/api-umbrella/admin-ui/app/controllers/stats/logs.js index 7b618de61..9fb48b6d2 100644 --- a/src/api-umbrella/admin-ui/app/controllers/stats/logs.js +++ b/src/api-umbrella/admin-ui/app/controllers/stats/logs.js @@ -8,6 +8,5 @@ export default Base.extend({ 'interval', 'query', 'search', - 'beta_analytics', ], }); diff --git a/src/api-umbrella/admin-ui/app/controllers/stats/map.js b/src/api-umbrella/admin-ui/app/controllers/stats/map.js index 44b0aaa62..1a41c0570 100644 --- a/src/api-umbrella/admin-ui/app/controllers/stats/map.js +++ b/src/api-umbrella/admin-ui/app/controllers/stats/map.js @@ -7,6 +7,5 @@ export default Base.extend({ 'query', 'search', 'region', - 'beta_analytics', ], }); diff --git a/src/api-umbrella/admin-ui/app/controllers/stats/users.js b/src/api-umbrella/admin-ui/app/controllers/stats/users.js index 9edc03586..f399b3f39 100644 --- a/src/api-umbrella/admin-ui/app/controllers/stats/users.js +++ b/src/api-umbrella/admin-ui/app/controllers/stats/users.js @@ -6,6 +6,5 @@ export default Base.extend({ 'end_at', 'query', 'search', - 'beta_analytics', ], }); diff --git a/src/api-umbrella/admin-ui/app/helpers/format-date.js b/src/api-umbrella/admin-ui/app/helpers/format-date.js index 79bffba2b..17e5c5e11 100644 --- a/src/api-umbrella/admin-ui/app/helpers/format-date.js +++ b/src/api-umbrella/admin-ui/app/helpers/format-date.js @@ -1,11 +1,12 @@ import { helper } from '@ember/component/helper'; -import moment from 'npm:moment-timezone'; +import isString from 'lodash-es/isString'; +import moment from 'moment-timezone'; export function formatDate(params) { let date = params[0]; let format = params[1]; - if(!format || !_.isString(format)) { + if(!format || !isString(format)) { format = 'YYYY-MM-DD HH:mm Z'; } diff --git a/src/api-umbrella/admin-ui/app/helpers/format-number.js b/src/api-umbrella/admin-ui/app/helpers/format-number.js new file mode 100644 index 000000000..673a527c1 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/helpers/format-number.js @@ -0,0 +1,10 @@ +import { helper } from '@ember/component/helper'; +import numeral from 'numeral'; + +export function formatNumber(params, args) { + const number = params[0]; + const format = (args || {}).format; + return numeral(number).format(format); +} + +export default helper(formatNumber); diff --git a/src/api-umbrella/admin-ui/app/helpers/inflect.js b/src/api-umbrella/admin-ui/app/helpers/inflect.js index 046893582..2080c1086 100644 --- a/src/api-umbrella/admin-ui/app/helpers/inflect.js +++ b/src/api-umbrella/admin-ui/app/helpers/inflect.js @@ -1,4 +1,5 @@ import { helper } from '@ember/component/helper'; +import inflection from 'inflection'; export function inflect(params) { let word = params[0]; diff --git a/src/api-umbrella/admin-ui/app/helpers/t.js b/src/api-umbrella/admin-ui/app/helpers/t.js index 6cf19e333..a5e71ca67 100644 --- a/src/api-umbrella/admin-ui/app/helpers/t.js +++ b/src/api-umbrella/admin-ui/app/helpers/t.js @@ -1,13 +1,13 @@ -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; +import { assign } from '@ember/polyfills'; import { helper } from '@ember/component/helper'; -import { merge } from '@ember/polyfills'; export function t(params, options) { let key = params[0]; // The options is an EmptyObject instance, which doesn't respond to // hasOwnProperty as I18n.t expects. // https://github.com/emberjs/ember.js/issues/14668 - const plainOptions = merge({}, options); + const plainOptions = assign({}, options); return I18n.t(key, plainOptions); } diff --git a/src/api-umbrella/admin-ui/app/index.html b/src/api-umbrella/admin-ui/app/index.html index be216322e..bc18ddf10 100644 --- a/src/api-umbrella/admin-ui/app/index.html +++ b/src/api-umbrella/admin-ui/app/index.html @@ -9,8 +9,8 @@ {{content-for "head"}} - - + + {{content-for "head-footer"}} @@ -31,8 +31,8 @@ {{content-for "body"}} - - + + {{content-for "body-footer"}} diff --git a/src/api-umbrella/admin-ui/app/initializers/bootbox.js b/src/api-umbrella/admin-ui/app/initializers/bootbox.js index 92647ac41..d795e7128 100644 --- a/src/api-umbrella/admin-ui/app/initializers/bootbox.js +++ b/src/api-umbrella/admin-ui/app/initializers/bootbox.js @@ -1,3 +1,5 @@ +import bootbox from 'bootbox'; + export function initialize() { bootbox.setDefaults({ animate: false, diff --git a/src/api-umbrella/admin-ui/app/initializers/bootstrap.js b/src/api-umbrella/admin-ui/app/initializers/bootstrap.js new file mode 100644 index 000000000..831e07e97 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/initializers/bootstrap.js @@ -0,0 +1,11 @@ +import 'bootstrap/js/dist/collapse'; +import 'bootstrap/js/dist/dropdown'; +import 'bootstrap/js/dist/modal'; + +export function initialize() { +} + +export default { + name: 'bootstrap', + initialize, +}; diff --git a/src/api-umbrella/admin-ui/app/initializers/echarts.js b/src/api-umbrella/admin-ui/app/initializers/echarts.js index 6bca13774..d8420aaca 100644 --- a/src/api-umbrella/admin-ui/app/initializers/echarts.js +++ b/src/api-umbrella/admin-ui/app/initializers/echarts.js @@ -1,4 +1,10 @@ -import echarts from 'npm:echarts'; +import 'echarts/lib/chart/line'; +import 'echarts/lib/chart/map'; +import 'echarts/lib/chart/scatter'; +import 'echarts/lib/component/geo'; +import 'echarts/lib/component/tooltip'; +import 'echarts/lib/component/visualMapContinuous'; +import echarts from 'echarts/lib/echarts'; export function initialize() { let colorPalette = [ @@ -37,21 +43,19 @@ export function initialize() { function mapCommon() { return { itemStyle: { - normal: { - color: 'transparent', - areaColor: '#f5f5f5', - borderColor: '#bbb', - }, - emphasis: { - borderColor: '#999', - borderWidth: 1, - }, + color: 'transparent', + areaColor: '#f5f5f5', + borderColor: '#bbb', }, label: { - normal: { - show: false, + show: false, + }, + emphasis: { + itemStyle: { + borderColor: '#999', + borderWidth: 1, }, - emphasis: { + label: { show: false, }, }, @@ -64,9 +68,7 @@ export function initialize() { color: colorPalette, }, areaStyle: { - normal: { - opacity: 0.2, - }, + opacity: 0.2, }, timeAxis: axisCommon(), logAxis: axisCommon(), @@ -76,11 +78,11 @@ export function initialize() { map: mapCommon(), scatter: { itemStyle: { - normal: { - borderColor: '#bbb', - borderWidth: 1, - }, - emphasis: { + borderColor: '#bbb', + borderWidth: 1, + }, + emphasis: { + itemStyle: { borderColor: '#666', borderWidth: 1, }, diff --git a/src/api-umbrella/admin-ui/app/initializers/fontawesome.js b/src/api-umbrella/admin-ui/app/initializers/fontawesome.js new file mode 100644 index 000000000..1ab119e22 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/initializers/fontawesome.js @@ -0,0 +1,10 @@ +import { dom } from '@fortawesome/fontawesome-svg-core'; + +export function initialize() { + dom.watch(); +} + +export default { + name: 'fontawesome', + initialize, +}; diff --git a/src/api-umbrella/admin-ui/app/initializers/pnotify.js b/src/api-umbrella/admin-ui/app/initializers/pnotify.js index ced111b4c..2670daef6 100644 --- a/src/api-umbrella/admin-ui/app/initializers/pnotify.js +++ b/src/api-umbrella/admin-ui/app/initializers/pnotify.js @@ -1,24 +1,14 @@ -import 'npm:pnotify/dist/pnotify.buttons'; -import 'npm:pnotify/dist/pnotify.mobile'; - -import PNotify from 'npm:pnotify'; +import 'pnotify/lib/es/PNotifyButtons'; +import 'pnotify/lib/es/PNotifyMobile'; +import PNotify from 'pnotify'; export function initialize() { - _.merge(PNotify.prototype.options, { - styling: 'bootstrap3', - width: '400px', - icon: false, - animation: 'none', - history: { - history: false, - }, - buttons: { - sticker: false, - }, - }); - - // Export the removeAll function as a global, for use in our test suite. - window.PNotifyRemoveAll = PNotify.removeAll; + PNotify.defaults.styling = 'bootstrap4'; + PNotify.defaults.width = '400px'; + PNotify.defaults.icon = false; + PNotify.defaults.icons = 'fontawesome5'; // Icons used for Buttons plugin. + PNotify.defaults.animation = 'none'; + PNotify.modules.Buttons.defaults.sticker = false; } export default { diff --git a/src/api-umbrella/admin-ui/app/initializers/qtip.js b/src/api-umbrella/admin-ui/app/initializers/qtip.js deleted file mode 100644 index fb8486291..000000000 --- a/src/api-umbrella/admin-ui/app/initializers/qtip.js +++ /dev/null @@ -1,69 +0,0 @@ -import $ from 'jquery'; - -export function initialize() { - $(document).on('click', 'a[rel=tooltip]', function(event) { - $(this).qtip({ - overwrite: false, - show: { - event: event.type, - ready: true, - solo: true, - }, - hide: { - event: 'unfocus', - }, - style: { - classes: 'qtip-bootstrap ' + $(this).data('tooltip-class'), - }, - position: { - viewport: true, - my: 'bottom left', - at: 'top center', - adjust: { - y: 2, - }, - }, - }, event); - - event.preventDefault(); - }); - - $(document).on('click', 'a[rel=popover]', function(event) { - $(this).qtip({ - overwrite: false, - show: { - event: event.type, - ready: true, - solo: true, - }, - hide: { - event: 'unfocus', - }, - content: { - text(event) { - let target = $(event.target).attr('href'); - let content = $(target).html(); - return content; - }, - }, - style: { - classes: 'qtip-bootstrap qtip-wide ' + $(this).data('tooltip-class'), - }, - position: { - viewport: false, - my: 'top left', - at: 'bottom center', - adjust: { - y: 2, - }, - }, - }, event); - - event.preventDefault(); - }); -} - -export default { - name: 'qtip', - initialize, -}; diff --git a/src/api-umbrella/admin-ui/app/initializers/test-disable-animations.js b/src/api-umbrella/admin-ui/app/initializers/test-disable-animations.js new file mode 100644 index 000000000..7d93fc5a8 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/initializers/test-disable-animations.js @@ -0,0 +1,45 @@ +import config from '../config/environment'; + +// Disable page animations when running in Capybara. This helps ensure the +// tests run more reliably and Capybara doesn't get confused about an element's +// visibility. +function ready() { + // Disable all CSS animations and transitions. + const style = document.createElement('style'); + style.type = 'text/css'; + style.innerHTML = '*, *:before, *:after {' + + '-webkit-transition: none !important;' + + '-moz-transition: none !important;' + + '-ms-transition: none !important;' + + '-o-transition: none !important;' + + 'transition: none !important;' + + '-webkit-animation: none !important;' + + '-moz-animation: none !important;' + + '-ms-animation: none !important;' + + '-o-animation: none !important;' + + 'animation: none !important;' + + '}'; + document.head.appendChild(style); + + // If jQuery is being used, then also disable it's animations. + if(window.jQuery) { + window.jQuery.support.transition = false; + window.jQuery.fx.off = true; + } +} + +export function initialize() { + if(config.integrationTestMode === true) { + // Setup now if document is already ready, or wait until document is ready. + if(document.readyState === 'interactive' || document.readyState === 'complete') { + ready(); + } else { + document.addEventListener('DOMContentLoaded', ready); + } + } +} + +export default { + name: 'test-disable-animations', + initialize, +}; diff --git a/src/api-umbrella/admin-ui/app/initializers/test-disable-fixed-header.js b/src/api-umbrella/admin-ui/app/initializers/test-disable-fixed-header.js new file mode 100644 index 000000000..b4f464ffd --- /dev/null +++ b/src/api-umbrella/admin-ui/app/initializers/test-disable-fixed-header.js @@ -0,0 +1,33 @@ +import config from '../config/environment'; + +// Disable the fixed, floating header in the admin. This occasionally causes +// problems with Poltergeist's scroll logic, since Poltergeist thinks it's +// scrolled an element to click into view, but then it discovers there's the +// navbar overlapping it, making it unclickable. +function ready() { + const style = document.createElement('style'); + style.type = 'text/css'; + style.innerHTML = 'body {' + + 'padding-top: 0px !important;' + + '}' + + '.fixed-top {' + + 'position: relative !important;' + + '}'; + document.head.appendChild(style); +} + +export function initialize() { + if(config.integrationTestMode === true) { + // Setup now if document is already ready, or wait until document is ready. + if(document.readyState === 'interactive' || document.readyState === 'complete') { + ready(); + } else { + document.addEventListener('DOMContentLoaded', ready); + } + } +} + +export default { + name: 'test-disable-fixed-header', + initialize, +}; diff --git a/src/api-umbrella/admin-ui/app/initializers/test-pnotify.js b/src/api-umbrella/admin-ui/app/initializers/test-pnotify.js new file mode 100644 index 000000000..248f9c105 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/initializers/test-pnotify.js @@ -0,0 +1,14 @@ +import PNotify from 'pnotify'; +import config from '../config/environment'; + +export function initialize() { + if(config.integrationTestMode === true) { + // Export the removeAll function as a global, for use in our test suite. + window.PNotifyRemoveAll = PNotify.removeAll; + } +} + +export default { + name: 'test-pnotify', + initialize, +}; diff --git a/src/api-umbrella/admin-ui/app/initializers/test-timekeeper.js b/src/api-umbrella/admin-ui/app/initializers/test-timekeeper.js new file mode 100644 index 000000000..ce3191162 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/initializers/test-timekeeper.js @@ -0,0 +1,13 @@ +import config from '../config/environment'; +import timekeeper from 'timekeeper'; + +export function initialize() { + if(config.integrationTestMode === true) { + window.timekeeper = timekeeper; + } +} + +export default { + name: 'test-timekeeper', + initialize, +}; diff --git a/src/api-umbrella/admin-ui/app/initializers/tooltips.js b/src/api-umbrella/admin-ui/app/initializers/tooltips.js new file mode 100644 index 000000000..06e0c8892 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/initializers/tooltips.js @@ -0,0 +1,23 @@ +import tippy from 'tippy.js' + +export function initialize() { + tippy('body', { + target: 'button.tooltip-trigger', + trigger: 'click', + interactive: true, + theme: 'light-border', + arrow: true, + onShow: (tip) => { + const contentSelector = tip.reference.getAttribute('data-tooltip-content-selector'); + if(contentSelector) { + const contentElement = document.querySelector(contentSelector); + tip.setContent(contentElement.innerHTML); + } + }, + }); +} + +export default { + name: 'tooltips', + initialize, +}; diff --git a/src/api-umbrella/admin-ui/app/instance-initializers/datatables.js b/src/api-umbrella/admin-ui/app/instance-initializers/datatables.js index 30cabc171..973b44b8e 100644 --- a/src/api-umbrella/admin-ui/app/instance-initializers/datatables.js +++ b/src/api-umbrella/admin-ui/app/instance-initializers/datatables.js @@ -1,8 +1,11 @@ +import 'datatables.net'; +import 'datatables.net-bs4'; import $ from 'jquery'; +import merge from 'lodash-es/merge'; export function initialize(appInstance) { // Defaults for DataTables. - _.merge($.fn.DataTable.defaults, { + merge($.fn.DataTable.defaults, { // Don't show the DataTables processing message. We'll handle the processing // message logic in preDrawCallback. processing: false, @@ -12,7 +15,7 @@ export function initialize(appInstance) { // Re-arrange how the table and surrounding fields (pagination, search, etc) // are laid out. - dom: 'rft<"row"<"col-sm-3 table-info"i><"col-sm-6 table-pagination"p><"col-sm-3 table-length"l>>', + dom: 'rft<"row"<"col-sm-3"i><"col-sm-6 table-pagination"p><"col-sm-3 table-length"l>>', language: { // Don't have an explicit label for the search field. Use a placeholder @@ -31,17 +34,27 @@ export function initialize(appInstance) { // // Set this early on during pre-draw so that the processing message shows // up for the first load. - $(this).DataTable().on('processing', _.bind(function(event, settings, processing) { + $(this).DataTable().on('processing', (event, settings, processing) => { if(processing) { appInstance.lookup('service:busy').show(); } else { appInstance.lookup('service:busy').hide(); } - }, this)); + }); this.customProcessingCallbackSet = true; } }, + + headerCallback(thead) { + $(thead).find('th:not(.sort-arrows-added)').append(''); + $(thead).find('th').addClass('sort-arrows-added'); + }, + }); + + merge($.fn.DataTable.ext.classes, { + sFilterInput: 'form-control', + sLengthSelect: 'custom-select form-control', }); } diff --git a/src/api-umbrella/admin-ui/app/mixins/confirmation.js b/src/api-umbrella/admin-ui/app/mixins/confirmation.js index 0542bbd9d..555ddf31f 100644 --- a/src/api-umbrella/admin-ui/app/mixins/confirmation.js +++ b/src/api-umbrella/admin-ui/app/mixins/confirmation.js @@ -1,5 +1,6 @@ import ConfirmationMixin from 'ember-onbeforeunload/mixins/confirmation'; import Mixin from '@ember/object/mixin' +import isEqual from 'lodash-es/isEqual'; export default Mixin.create(ConfirmationMixin, { afterModel(model) { @@ -48,7 +49,7 @@ export default Mixin.create(ConfirmationMixin, { } else { let initialSerialized = record.get('_confirmationRecordInitialSerialized'); let currentSerialized = record.serialize(); - return !_.isEqual(currentSerialized, initialSerialized); + return !isEqual(currentSerialized, initialSerialized); } }, }); diff --git a/src/api-umbrella/admin-ui/app/mixins/save.js b/src/api-umbrella/admin-ui/app/mixins/save.js index 60179be8d..c55ad8ff6 100644 --- a/src/api-umbrella/admin-ui/app/mixins/save.js +++ b/src/api-umbrella/admin-ui/app/mixins/save.js @@ -1,42 +1,45 @@ -import $ from 'jquery'; +import LoadingButton from 'api-umbrella-admin-ui/utils/loading-button'; import Mixin from '@ember/object/mixin' -import PNotify from 'npm:pnotify'; +import PNotify from 'pnotify'; +import bootbox from 'bootbox'; import { inject } from '@ember/service'; +import isFunction from 'lodash-es/isFunction'; +import scrollTo from 'jquery.scrollto'; export default Mixin.create({ router: inject(), - scrollToErrors() { - $('#save_button').button('reset'); - $.scrollTo('#error_messages', { offset: -60, duration: 200 }); + scrollToErrors(button) { + LoadingButton.reset(button); + scrollTo('#error_messages', { offset: -60, duration: 200 }); }, afterSaveComplete(options, button) { - button.button('reset'); - new PNotify({ - type: 'success', + LoadingButton.reset(button); + PNotify.success({ title: 'Saved', - text: (_.isFunction(options.message)) ? options.message(this.get('model')) : options.message, + text: (isFunction(options.message)) ? options.message(this.model) : options.message, + textTrusted: true, }); - this.get('router').transitionTo(options.transitionToRoute); + this.router.transitionTo(options.transitionToRoute); }, saveRecord(options) { - let button = $('#save_button'); - button.button('loading'); + const button = this.element.querySelector('.save-button'); + LoadingButton.loading(button); this.setProperties({ 'model.clientErrors': [], 'model.serverErrors': [], }); - this.get('model').validate().then(function() { + this.model.validate().then(function() { if(this.get('model.validations.isValid') === false) { this.set('model.clientErrors', this.get('model.validations.errors')); - this.scrollToErrors(); + this.scrollToErrors(button); } else { - this.get('model').save().then(function() { + this.model.save().then(function() { if(options.afterSave) { options.afterSave(this.afterSaveComplete.bind(this, options, button)); } else { @@ -48,10 +51,12 @@ export default Mixin.create({ if(error && error.errors) { this.set('model.serverErrors', error.errors); } else { + // eslint-disable-next-line no-console + console.error('Unexpected save error: ', error); this.set('model.serverErrors', [{ message: 'Unexpected error' }]); } - this.scrollToErrors(); + this.scrollToErrors(button); }.bind(this)); } }.bind(this)); @@ -60,14 +65,14 @@ export default Mixin.create({ destroyRecord(options) { bootbox.confirm(options.prompt, function(result) { if(result) { - this.get('model').destroyRecord().then(function() { - new PNotify({ - type: 'success', + this.model.destroyRecord().then(function() { + PNotify.success({ title: 'Deleted', - text: (_.isFunction(options.message)) ? options.message(this.get('model')) : options.message, + text: (isFunction(options.message)) ? options.message(this.model) : options.message, + textTrusted: true, }); - this.get('router').transitionTo(options.transitionToRoute); + this.router.transitionTo(options.transitionToRoute); }.bind(this), function(response) { bootbox.alert('Unexpected error deleting record: ' + response.responseText); }); diff --git a/src/api-umbrella/admin-ui/app/mixins/sortable.js b/src/api-umbrella/admin-ui/app/mixins/sortable.js index 018e86bbf..2683474a5 100644 --- a/src/api-umbrella/admin-ui/app/mixins/sortable.js +++ b/src/api-umbrella/admin-ui/app/mixins/sortable.js @@ -1,3 +1,4 @@ +import 'jquery-ui/ui/widgets/sortable'; import $ from 'jquery'; import Mixin from '@ember/object/mixin' import { computed } from '@ember/object'; @@ -10,7 +11,7 @@ export default Mixin.create({ }), updateSortOrder(indexes) { - this.get('sortableCollection').forEach(function(record) { + this.sortableCollection.forEach(function(record) { let index = indexes[guidFor(record)]; record.set('sortOrder', index); }); diff --git a/src/api-umbrella/admin-ui/app/mixins/uncached-model.js b/src/api-umbrella/admin-ui/app/mixins/uncached-model.js index 427b3fddf..61d8da307 100644 --- a/src/api-umbrella/admin-ui/app/mixins/uncached-model.js +++ b/src/api-umbrella/admin-ui/app/mixins/uncached-model.js @@ -1,4 +1,5 @@ import Mixin from '@ember/object/mixin' +import { run } from '@ember/runloop'; export default Mixin.create({ // Call before fetching a model to clear any client-side cache data. @@ -31,6 +32,11 @@ export default Mixin.create({ // https://github.com/emberjs/data/issues/4564 // https://github.com/emberjs/data/issues/4595 clearStoreCache() { - this.get('store').unloadAll(); + // Must explicitly wrap in run loop or else the _idToModel mapping is still + // present (at least in development, but oddly not in production mode). + // Semi-related: https://github.com/emberjs/data/issues/5041 + run(() => { + this.store.unloadAll(); + }); }, }); diff --git a/src/api-umbrella/admin-ui/app/models/api-scope.js b/src/api-umbrella/admin-ui/app/models/api-scope.js index 17917f761..3be5dd470 100644 --- a/src/api-umbrella/admin-ui/app/models/api-scope.js +++ b/src/api-umbrella/admin-ui/app/models/api-scope.js @@ -1,7 +1,7 @@ import { buildValidations, validator } from 'ember-cp-validations'; import DS from 'ember-data'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; import { computed } from '@ember/object'; const Validations = buildValidations({ @@ -32,7 +32,7 @@ export default DS.Model.extend(Validations, { updater: DS.attr(), displayName: computed('name', 'host', 'pathPrefix', function() { - return this.get('name') + ' - ' + this.get('host') + this.get('pathPrefix'); + return this.name + ' - ' + this.host + this.pathPrefix; }), }).reopenClass({ urlRoot: '/api-umbrella/v1/api_scopes', diff --git a/src/api-umbrella/admin-ui/app/models/api-user.js b/src/api-umbrella/admin-ui/app/models/api-user.js index fb933dfb0..902627616 100644 --- a/src/api-umbrella/admin-ui/app/models/api-user.js +++ b/src/api-umbrella/admin-ui/app/models/api-user.js @@ -1,6 +1,7 @@ import { buildValidations, validator } from 'ember-cp-validations'; import DS from 'ember-data'; +import compact from 'lodash-es/compact'; import { computed } from '@ember/object'; const Validations = buildValidations({ @@ -42,19 +43,19 @@ export default DS.Model.extend(Validations, { }, setDefaults() { - if(this.get('throttleByIp') === undefined) { + if(this.throttleByIp === undefined) { this.set('throttleByIp', false); } - if(this.get('enabled') === undefined) { + if(this.enabled === undefined) { this.set('enabled', true); } - if(!this.get('settings')) { - this.set('settings', this.get('store').createRecord('api/settings')); + if(!this.settings) { + this.set('settings', this.store.createRecord('api/settings')); } - if(!this.get('registrationSource') && this.get('isNew')) { + if(!this.registrationSource && this.isNew) { this.set('registrationSource', 'web_admin'); } }, @@ -62,13 +63,13 @@ export default DS.Model.extend(Validations, { rolesString: computed('roles', { get() { let rolesString = ''; - if(this.get('roles')) { - rolesString = this.get('roles').join(','); + if(this.roles) { + rolesString = this.roles.join(','); } return rolesString; }, set(key, value) { - let roles = _.compact(value.split(',')); + let roles = compact(value.split(',')); if(roles.length === 0) { roles = null; } this.set('roles', roles); return value; diff --git a/src/api-umbrella/admin-ui/app/models/api.js b/src/api-umbrella/admin-ui/app/models/api.js index 137989871..76beed394 100644 --- a/src/api-umbrella/admin-ui/app/models/api.js +++ b/src/api-umbrella/admin-ui/app/models/api.js @@ -1,7 +1,7 @@ import { buildValidations, validator } from 'ember-cp-validations'; import DS from 'ember-data'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; import { computed } from '@ember/object'; const Validations = buildValidations({ @@ -54,17 +54,17 @@ export default DS.Model.extend(Validations, { }, setDefaults() { - if(!this.get('settings')) { - this.set('settings', this.get('store').createRecord('api/settings')); + if(!this.settings) { + this.set('settings', this.store.createRecord('api/settings')); } }, exampleIncomingUrlRoot: computed('frontendHost', function() { - return 'https://' + (this.get('frontendHost') || ''); + return 'https://' + (this.frontendHost || ''); }), exampleOutgoingUrlRoot: computed('backendProtocol', 'backendHost', 'fontendHost', function() { - return this.get('backendProtocol') + '://' + (this.get('backendHost') || this.get('frontendHost') || ''); + return this.backendProtocol + '://' + (this.backendHost || this.frontendHost || ''); }), }).reopenClass({ urlRoot: '/api-umbrella/v1/apis', diff --git a/src/api-umbrella/admin-ui/app/models/api/rate-limit.js b/src/api-umbrella/admin-ui/app/models/api/rate-limit.js index b541b8d7b..ec9bf80dc 100644 --- a/src/api-umbrella/admin-ui/app/models/api/rate-limit.js +++ b/src/api-umbrella/admin-ui/app/models/api/rate-limit.js @@ -1,6 +1,7 @@ +import { computed, observer } from '@ember/object'; import DS from 'ember-data'; -import moment from 'npm:moment-timezone'; -import { observer } from '@ember/object'; +import moment from 'moment-timezone'; +import uniqueId from 'lodash-es/uniqueId'; export default DS.Model.extend({ duration: DS.attr('number'), @@ -14,7 +15,7 @@ export default DS.Model.extend({ }, setDefaults() { - let duration = this.get('duration'); + let duration = this.duration; if(duration) { let days = duration / 86400000; let hours = duration / 3600000; @@ -46,10 +47,14 @@ export default DS.Model.extend({ }, durationInUnitsDidChange: observer('durationInUnits', 'durationUnits', function() { - if(this.get('durationUnits')) { - let inUnits = parseInt(this.get('durationInUnits'), 10); - let units = this.get('durationUnits'); + if(this.durationUnits) { + let inUnits = parseInt(this.durationInUnits, 10); + let units = this.durationUnits; this.set('duration', moment.duration(inUnits, units).asMilliseconds()); } }), + + uniqueId: computed(function() { + return uniqueId('rate_limit_'); + }), }); diff --git a/src/api-umbrella/admin-ui/app/models/api/server.js b/src/api-umbrella/admin-ui/app/models/api/server.js index 5b93c873a..9807c0f17 100644 --- a/src/api-umbrella/admin-ui/app/models/api/server.js +++ b/src/api-umbrella/admin-ui/app/models/api/server.js @@ -1,7 +1,8 @@ import { buildValidations, validator } from 'ember-cp-validations'; import DS from 'ember-data'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; +import compact from 'lodash-es/compact'; import { computed } from '@ember/object'; const Validations = buildValidations({ @@ -23,7 +24,7 @@ export default DS.Model.extend(Validations, { port: DS.attr('number'), hostWithPort: computed('host', 'port', function() { - return _.compact([this.get('host'), this.get('port')]).join(':'); + return compact([this.host, this.port]).join(':'); }), }).reopenClass({ validationClass: Validations, diff --git a/src/api-umbrella/admin-ui/app/models/api/settings.js b/src/api-umbrella/admin-ui/app/models/api/settings.js index 234c6aa09..92e020c56 100644 --- a/src/api-umbrella/admin-ui/app/models/api/settings.js +++ b/src/api-umbrella/admin-ui/app/models/api/settings.js @@ -2,6 +2,7 @@ import EmberObject, { computed, observer } from '@ember/object'; import { A } from '@ember/array'; import DS from 'ember-data'; +import compact from 'lodash-es/compact'; export default DS.Model.extend({ appendQueryString: DS.attr(), @@ -36,17 +37,29 @@ export default DS.Model.extend({ }, setDefaults() { - if(this.get('rateLimitMode') === undefined) { + if(this.requireHttps === undefined) { + this.set('requireHttps', null); + } + + if(this.disableApiKey === undefined) { + this.set('disableApiKey', null); + } + + if(this.apiKeyVerificationLevel === undefined) { + this.set('apiKeyVerificationLevel', null); + } + + if(this.rateLimitMode === undefined) { this.set('rateLimitMode', null); } // Make sure at least an empty object exists so the form builder can dive // into this section even when there's no pre-existing data. - if(!this.get('errorTemplates')) { + if(!this.errorTemplates) { this.set('errorTemplates', EmberObject.create({})); } - if(!this.get('errorDataYamlStrings')) { + if(!this.errorDataYamlStrings) { this.set('errorDataYamlStrings', EmberObject.create({})); } }, @@ -54,13 +67,13 @@ export default DS.Model.extend({ requiredRolesString: computed('requiredRoles', { get() { let rolesString = ''; - if(this.get('requiredRoles')) { - rolesString = this.get('requiredRoles').join(','); + if(this.requiredRoles) { + rolesString = this.requiredRoles.join(','); } return rolesString; }, set(key, value) { - let roles = _.compact(value.split(',')); + let roles = compact(value.split(',')); if(roles.length === 0) { roles = null; } this.set('requiredRoles', roles); return value; @@ -70,13 +83,13 @@ export default DS.Model.extend({ allowedIpsString: computed('allowedIps', { get() { let allowedIpsString = ''; - if(this.get('allowedIps')) { - allowedIpsString = this.get('allowedIps').join('\n'); + if(this.allowedIps) { + allowedIpsString = this.allowedIps.join('\n'); } return allowedIpsString; }, set(key, value) { - let ips = _.compact(value.split(/[\r\n]+/)); + let ips = compact(value.split(/[\r\n]+/)); if(ips.length === 0) { ips = null; } this.set('allowedIps', ips); return value; @@ -86,13 +99,13 @@ export default DS.Model.extend({ allowedReferersString: computed('allowedReferers', { get() { let allowedReferersString = ''; - if(this.get('allowedReferers')) { - allowedReferersString = this.get('allowedReferers').join('\n'); + if(this.allowedReferers) { + allowedReferersString = this.allowedReferers.join('\n'); } return allowedReferersString; }, set(key, value) { - let referers = _.compact(value.split(/[\r\n]+/)); + let referers = compact(value.split(/[\r\n]+/)); if(referers.length === 0) { referers = null; } this.set('allowedReferers', referers); return value; @@ -101,27 +114,22 @@ export default DS.Model.extend({ passApiKey: computed('passApiKeyHeader', 'passApiKeyQueryParam', function() { let options = A([]); - if(this.get('passApiKeyHeader')) { + if(this.passApiKeyHeader) { options.pushObject('header'); } - if(this.get('passApiKeyQueryParam')) { + if(this.passApiKeyQueryParam) { options.pushObject('param'); } return options; }), passApiKeyDidChange: observer('passApiKey.@each', function() { - let options = this.get('passApiKey'); + let options = this.passApiKey; this.set('passApiKeyHeader', options.includes('header')); this.set('passApiKeyQueryParam', options.includes('param')); }), isRateLimitModeCustom: computed('rateLimitMode', function() { - return (this.get('rateLimitMode') === 'custom' - || this.get('rateLimitMode') === 'custom-header'); - }), - - hasRateLimitCostHeader: computed('rateLimitMode', function() { - return (this.get('rateLimitMode') === 'custom-header'); + return this.rateLimitMode === 'custom'; }), }); diff --git a/src/api-umbrella/admin-ui/app/models/api/sub-settings.js b/src/api-umbrella/admin-ui/app/models/api/sub-settings.js index 83e436ac3..b754f85a8 100644 --- a/src/api-umbrella/admin-ui/app/models/api/sub-settings.js +++ b/src/api-umbrella/admin-ui/app/models/api/sub-settings.js @@ -24,8 +24,8 @@ export default DS.Model.extend(Validations, { }, setDefaults() { - if(!this.get('settings')) { - this.set('settings', this.get('store').createRecord('api/settings')); + if(!this.settings) { + this.set('settings', this.store.createRecord('api/settings')); } }, }).reopenClass({ diff --git a/src/api-umbrella/admin-ui/app/models/api/url-match.js b/src/api-umbrella/admin-ui/app/models/api/url-match.js index 50d83eeb4..3bea1de80 100644 --- a/src/api-umbrella/admin-ui/app/models/api/url-match.js +++ b/src/api-umbrella/admin-ui/app/models/api/url-match.js @@ -1,7 +1,7 @@ import { buildValidations, validator } from 'ember-cp-validations'; import DS from 'ember-data'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; import { computed } from '@ember/object'; const Validations = buildValidations({ @@ -27,7 +27,7 @@ export default DS.Model.extend(Validations, { backendPrefix: DS.attr(), backendPrefixWithDefault: computed('backendPrefix', 'frontendPrefix', function() { - return this.get('backendPrefix') || this.get('frontendPrefix'); + return this.backendPrefix || this.frontendPrefix; }), }).reopenClass({ validationClass: Validations, diff --git a/src/api-umbrella/admin-ui/app/models/config-pending-changes.js b/src/api-umbrella/admin-ui/app/models/config-pending-changes.js index fb8d22df2..7a273fc92 100644 --- a/src/api-umbrella/admin-ui/app/models/config-pending-changes.js +++ b/src/api-umbrella/admin-ui/app/models/config-pending-changes.js @@ -16,7 +16,7 @@ ConfigPendingChanges.reopenClass({ url: this.urlRoot, data: params, }).then(function(data) { - resolve(new ConfigPendingChanges(data)); + resolve(ConfigPendingChanges.create(data)); }, function(data) { reject(data.responseText); }); diff --git a/src/api-umbrella/admin-ui/app/models/stats/drilldown.js b/src/api-umbrella/admin-ui/app/models/stats/drilldown.js index 8d096436e..61eafb320 100644 --- a/src/api-umbrella/admin-ui/app/models/stats/drilldown.js +++ b/src/api-umbrella/admin-ui/app/models/stats/drilldown.js @@ -16,7 +16,7 @@ Drilldown.reopenClass({ url: this.urlRoot, data: params, }).then(function(data) { - resolve(new Drilldown(data)); + resolve(Drilldown.create(data)); }, function(data) { reject(data.responseText); }); diff --git a/src/api-umbrella/admin-ui/app/models/stats/logs.js b/src/api-umbrella/admin-ui/app/models/stats/logs.js index b33dccb34..c3fe413d8 100644 --- a/src/api-umbrella/admin-ui/app/models/stats/logs.js +++ b/src/api-umbrella/admin-ui/app/models/stats/logs.js @@ -19,7 +19,7 @@ Logs.reopenClass({ url: this.urlRoot, data: params, }).then(function(data) { - resolve(new Logs(data)); + resolve(Logs.create(data)); }, function(data) { reject(data.responseText); }); diff --git a/src/api-umbrella/admin-ui/app/models/stats/map.js b/src/api-umbrella/admin-ui/app/models/stats/map.js index 4de9b0a43..fce34f730 100644 --- a/src/api-umbrella/admin-ui/app/models/stats/map.js +++ b/src/api-umbrella/admin-ui/app/models/stats/map.js @@ -19,7 +19,7 @@ Map.reopenClass({ url: this.urlRoot, data: params, }).then(function(data) { - resolve(new Map(data)); + resolve(Map.create(data)); }, function(data) { reject(data.responseText); }); diff --git a/src/api-umbrella/admin-ui/app/models/website-backend.js b/src/api-umbrella/admin-ui/app/models/website-backend.js index 356ee486c..0b33b83bb 100644 --- a/src/api-umbrella/admin-ui/app/models/website-backend.js +++ b/src/api-umbrella/admin-ui/app/models/website-backend.js @@ -1,7 +1,7 @@ import { buildValidations, validator } from 'ember-cp-validations'; import DS from 'ember-data'; -import I18n from 'npm:i18n-js'; +import I18n from 'i18n-js'; const Validations = buildValidations({ frontendHost: [ diff --git a/src/api-umbrella/admin-ui/app/routes/admin-groups/base.js b/src/api-umbrella/admin-ui/app/routes/admin-groups/base.js index c749c1ce9..d7aa90d0d 100644 --- a/src/api-umbrella/admin-ui/app/routes/admin-groups/base.js +++ b/src/api-umbrella/admin-ui/app/routes/admin-groups/base.js @@ -6,7 +6,7 @@ export default Route.extend(AuthenticatedRouteMixin, { setupController(controller, model) { controller.set('model', model); - $('ul.nav li').removeClass('active'); - $('ul.nav li.nav-users').addClass('active'); + $('ul.navbar-nav li').removeClass('active'); + $('ul.navbar-nav li.nav-users').addClass('active'); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/admin-groups/edit.js b/src/api-umbrella/admin-ui/app/routes/admin-groups/edit.js index d686fdb33..39f1cbfb6 100644 --- a/src/api-umbrella/admin-ui/app/routes/admin-groups/edit.js +++ b/src/api-umbrella/admin-ui/app/routes/admin-groups/edit.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model(params) { this.clearStoreCache(); - return this.fetchModels(this.get('store').findRecord('admin-group', params.admin_group_id, { reload: true })); + return this.fetchModels(this.store.findRecord('admin-group', params.admin_group_id, { reload: true })); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/admin-groups/form.js b/src/api-umbrella/admin-ui/app/routes/admin-groups/form.js index de481987f..179ff89ec 100644 --- a/src/api-umbrella/admin-ui/app/routes/admin-groups/form.js +++ b/src/api-umbrella/admin-ui/app/routes/admin-groups/form.js @@ -8,8 +8,8 @@ export default Base.extend(Confirmation, UncachedModel, { fetchModels(record) { return hash({ record: record, - apiScopeOptions: this.get('store').findAll('api-scope', { reload: true }), - permissionOptions: this.get('store').findAll('admin-permission', { reload: true }), + apiScopeOptions: this.store.findAll('api-scope', { reload: true }), + permissionOptions: this.store.findAll('admin-permission', { reload: true }), }); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/admin-groups/new.js b/src/api-umbrella/admin-ui/app/routes/admin-groups/new.js index dce11b635..5a5eb8113 100644 --- a/src/api-umbrella/admin-ui/app/routes/admin-groups/new.js +++ b/src/api-umbrella/admin-ui/app/routes/admin-groups/new.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model() { this.clearStoreCache(); - return this.fetchModels(this.get('store').createRecord('admin-group')); + return this.fetchModels(this.store.createRecord('admin-group')); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/admins/base.js b/src/api-umbrella/admin-ui/app/routes/admins/base.js index c749c1ce9..d7aa90d0d 100644 --- a/src/api-umbrella/admin-ui/app/routes/admins/base.js +++ b/src/api-umbrella/admin-ui/app/routes/admins/base.js @@ -6,7 +6,7 @@ export default Route.extend(AuthenticatedRouteMixin, { setupController(controller, model) { controller.set('model', model); - $('ul.nav li').removeClass('active'); - $('ul.nav li.nav-users').addClass('active'); + $('ul.navbar-nav li').removeClass('active'); + $('ul.navbar-nav li.nav-users').addClass('active'); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/admins/edit.js b/src/api-umbrella/admin-ui/app/routes/admins/edit.js index 3016cb314..3d0a1f569 100644 --- a/src/api-umbrella/admin-ui/app/routes/admins/edit.js +++ b/src/api-umbrella/admin-ui/app/routes/admins/edit.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model(params) { this.clearStoreCache(); - return this.fetchModels(this.get('store').findRecord('admin', params.admin_id, { reload: true })); + return this.fetchModels(this.store.findRecord('admin', params.admin_id, { reload: true })); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/admins/form.js b/src/api-umbrella/admin-ui/app/routes/admins/form.js index f5850660c..2b1bfe2ea 100644 --- a/src/api-umbrella/admin-ui/app/routes/admins/form.js +++ b/src/api-umbrella/admin-ui/app/routes/admins/form.js @@ -8,7 +8,7 @@ export default Base.extend(Confirmation, UncachedModel, { fetchModels(record) { return hash({ record: record, - groupOptions: this.get('store').findAll('admin-group', { reload: true }), + groupOptions: this.store.findAll('admin-group', { reload: true }), }); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/admins/new.js b/src/api-umbrella/admin-ui/app/routes/admins/new.js index 7436ced94..79d7e2455 100644 --- a/src/api-umbrella/admin-ui/app/routes/admins/new.js +++ b/src/api-umbrella/admin-ui/app/routes/admins/new.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model() { this.clearStoreCache(); - return this.fetchModels(this.get('store').createRecord('admin', { sendInviteEmail: true })); + return this.fetchModels(this.store.createRecord('admin', { sendInviteEmail: true })); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/api-scopes/base.js b/src/api-umbrella/admin-ui/app/routes/api-scopes/base.js index c749c1ce9..d7aa90d0d 100644 --- a/src/api-umbrella/admin-ui/app/routes/api-scopes/base.js +++ b/src/api-umbrella/admin-ui/app/routes/api-scopes/base.js @@ -6,7 +6,7 @@ export default Route.extend(AuthenticatedRouteMixin, { setupController(controller, model) { controller.set('model', model); - $('ul.nav li').removeClass('active'); - $('ul.nav li.nav-users').addClass('active'); + $('ul.navbar-nav li').removeClass('active'); + $('ul.navbar-nav li.nav-users').addClass('active'); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/api-scopes/edit.js b/src/api-umbrella/admin-ui/app/routes/api-scopes/edit.js index c0b5a54a6..4a9c41dc9 100644 --- a/src/api-umbrella/admin-ui/app/routes/api-scopes/edit.js +++ b/src/api-umbrella/admin-ui/app/routes/api-scopes/edit.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model(params) { this.clearStoreCache(); - return this.get('store').findRecord('api-scope', params.api_scope_id, { reload: true }); + return this.store.findRecord('api-scope', params.api_scope_id, { reload: true }); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/api-scopes/new.js b/src/api-umbrella/admin-ui/app/routes/api-scopes/new.js index 0ed978e43..e0c4ed291 100644 --- a/src/api-umbrella/admin-ui/app/routes/api-scopes/new.js +++ b/src/api-umbrella/admin-ui/app/routes/api-scopes/new.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model() { this.clearStoreCache(); - return this.get('store').createRecord('api-scope'); + return this.store.createRecord('api-scope'); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/api-users/base.js b/src/api-umbrella/admin-ui/app/routes/api-users/base.js index c749c1ce9..d7aa90d0d 100644 --- a/src/api-umbrella/admin-ui/app/routes/api-users/base.js +++ b/src/api-umbrella/admin-ui/app/routes/api-users/base.js @@ -6,7 +6,7 @@ export default Route.extend(AuthenticatedRouteMixin, { setupController(controller, model) { controller.set('model', model); - $('ul.nav li').removeClass('active'); - $('ul.nav li.nav-users').addClass('active'); + $('ul.navbar-nav li').removeClass('active'); + $('ul.navbar-nav li.nav-users').addClass('active'); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/api-users/edit.js b/src/api-umbrella/admin-ui/app/routes/api-users/edit.js index aa33b403e..056a2153d 100644 --- a/src/api-umbrella/admin-ui/app/routes/api-users/edit.js +++ b/src/api-umbrella/admin-ui/app/routes/api-users/edit.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model(params) { this.clearStoreCache(); - return this.fetchModels(this.get('store').findRecord('api-user', params.api_user_id, { reload: true })); + return this.fetchModels(this.store.findRecord('api-user', params.api_user_id, { reload: true })); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/api-users/form.js b/src/api-umbrella/admin-ui/app/routes/api-users/form.js index 3573dd1d3..3c8195449 100644 --- a/src/api-umbrella/admin-ui/app/routes/api-users/form.js +++ b/src/api-umbrella/admin-ui/app/routes/api-users/form.js @@ -8,7 +8,7 @@ export default Base.extend(Confirmation, UncachedModel, { fetchModels(record) { return hash({ record: record, - roleOptions: this.get('store').findAll('api-user-role', { reload: true }), + roleOptions: this.store.findAll('api-user-role', { reload: true }), }); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/api-users/new.js b/src/api-umbrella/admin-ui/app/routes/api-users/new.js index 4a192424c..d63bc9c84 100644 --- a/src/api-umbrella/admin-ui/app/routes/api-users/new.js +++ b/src/api-umbrella/admin-ui/app/routes/api-users/new.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model() { this.clearStoreCache(); - return this.fetchModels(this.get('store').createRecord('api-user')); + return this.fetchModels(this.store.createRecord('api-user')); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/apis/base.js b/src/api-umbrella/admin-ui/app/routes/apis/base.js index ebe15b271..b335cb0b1 100644 --- a/src/api-umbrella/admin-ui/app/routes/apis/base.js +++ b/src/api-umbrella/admin-ui/app/routes/apis/base.js @@ -6,7 +6,7 @@ export default Route.extend(AuthenticatedRouteMixin, { setupController(controller, model) { controller.set('model', model); - $('ul.nav li').removeClass('active'); - $('ul.nav li.nav-config').addClass('active'); + $('ul.navbar-nav li').removeClass('active'); + $('ul.navbar-nav li.nav-config').addClass('active'); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/apis/edit.js b/src/api-umbrella/admin-ui/app/routes/apis/edit.js index d8210a619..721133995 100644 --- a/src/api-umbrella/admin-ui/app/routes/apis/edit.js +++ b/src/api-umbrella/admin-ui/app/routes/apis/edit.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model(params) { this.clearStoreCache(); - return this.fetchModels(this.get('store').findRecord('api', params.api_id, { reload: true })); + return this.fetchModels(this.store.findRecord('api', params.api_id, { reload: true })); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/apis/form.js b/src/api-umbrella/admin-ui/app/routes/apis/form.js index 3573dd1d3..3c8195449 100644 --- a/src/api-umbrella/admin-ui/app/routes/apis/form.js +++ b/src/api-umbrella/admin-ui/app/routes/apis/form.js @@ -8,7 +8,7 @@ export default Base.extend(Confirmation, UncachedModel, { fetchModels(record) { return hash({ record: record, - roleOptions: this.get('store').findAll('api-user-role', { reload: true }), + roleOptions: this.store.findAll('api-user-role', { reload: true }), }); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/apis/new.js b/src/api-umbrella/admin-ui/app/routes/apis/new.js index 022d9c8b3..c21fd7b73 100644 --- a/src/api-umbrella/admin-ui/app/routes/apis/new.js +++ b/src/api-umbrella/admin-ui/app/routes/apis/new.js @@ -3,7 +3,7 @@ import Form from './form'; export default Form.extend({ model() { this.clearStoreCache(); - return this.fetchModels(this.get('store').createRecord('api', { + return this.fetchModels(this.store.createRecord('api', { frontendHost: location.hostname, })); }, diff --git a/src/api-umbrella/admin-ui/app/routes/application.js b/src/api-umbrella/admin-ui/app/routes/application.js index 8e897edcb..3f36546f7 100644 --- a/src/api-umbrella/admin-ui/app/routes/application.js +++ b/src/api-umbrella/admin-ui/app/routes/application.js @@ -1,6 +1,7 @@ import ApplicationRouteMixin from 'ember-simple-auth/mixins/application-route-mixin'; import Route from '@ember/routing/route'; import { inject } from '@ember/service'; +import isString from 'lodash-es/isString'; import { observer } from '@ember/object'; export default Route.extend(ApplicationRouteMixin, { @@ -15,9 +16,9 @@ export default Route.extend(ApplicationRouteMixin, { attemptedTransitionChange: observer('session.attemptedTransition', function() { const attemptedTransition = this.get('session.attemptedTransition'); if(attemptedTransition) { - this.get('session').set('data.attemptedTransitionUrl', attemptedTransition.intent.url); + this.session.set('data.attemptedTransitionUrl', attemptedTransition.intent.url); } else { - this.get('session').set('data.attemptedTransitionUrl', null); + this.session.set('data.attemptedTransitionUrl', null); } }), @@ -31,15 +32,15 @@ export default Route.extend(ApplicationRouteMixin, { if(attemptedTransitionUrl) { this.transitionTo(attemptedTransitionUrl); this.set('session.attemptedTransition', null); - this.get('session').set('data.attemptedTransitionUrl', null); + this.session.set('data.attemptedTransitionUrl', null); } else { - this.transitionTo(this.get('routeAfterAuthentication')); + this.transitionTo(this.routeAfterAuthentication); } }, actions: { loading(transition) { - let busy = this.get('busy'); + let busy = this.busy; busy.show(); transition.promise.finally(function() { busy.hide(); @@ -57,13 +58,13 @@ export default Route.extend(ApplicationRouteMixin, { errorMessage = error; // Very long text error messages can seem to hang some of the console // tools, so truncate the messages. - if(_.isString(errorMessage)) { + if(isString(errorMessage)) { errorMessage = errorMessage.substring(0, 1000); } } // eslint-disable-next-line no-console console.error(errorMessage); - this.get('busy').hide(); + this.busy.hide(); return this.intermediateTransitionTo('error'); } }, diff --git a/src/api-umbrella/admin-ui/app/routes/config/publish.js b/src/api-umbrella/admin-ui/app/routes/config/publish.js index 8ae82bf9c..27f944433 100644 --- a/src/api-umbrella/admin-ui/app/routes/config/publish.js +++ b/src/api-umbrella/admin-ui/app/routes/config/publish.js @@ -11,7 +11,7 @@ export default Route.extend(AuthenticatedRouteMixin, { setupController(controller, model) { controller.set('model', model); - $('ul.nav li').removeClass('active'); - $('ul.nav li.nav-config').addClass('active'); + $('ul.navbar-nav li').removeClass('active'); + $('ul.navbar-nav li.nav-config').addClass('active'); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/login.js b/src/api-umbrella/admin-ui/app/routes/login.js index 51bebff55..869bfffdd 100644 --- a/src/api-umbrella/admin-ui/app/routes/login.js +++ b/src/api-umbrella/admin-ui/app/routes/login.js @@ -7,7 +7,7 @@ export default Route.extend(UnauthenticatedRouteMixin, { }, authenticate() { - this.get('session').authenticate('authenticator:devise-server-side').catch((error) => { + this.session.authenticate('authenticator:devise-server-side').catch((error) => { if(error !== 'unexpected_error') { window.location.href = '/admin/login'; } diff --git a/src/api-umbrella/admin-ui/app/routes/stats/base.js b/src/api-umbrella/admin-ui/app/routes/stats/base.js index b247ba965..3a2ef793a 100644 --- a/src/api-umbrella/admin-ui/app/routes/stats/base.js +++ b/src/api-umbrella/admin-ui/app/routes/stats/base.js @@ -1,18 +1,21 @@ import $ from 'jquery'; import AuthenticatedRouteMixin from 'ember-simple-auth/mixins/authenticated-route-mixin'; import Route from '@ember/routing/route'; -import moment from 'npm:moment-timezone'; +import bootbox from 'bootbox'; +import cloneDeep from 'lodash-es/cloneDeep'; +import moment from 'moment-timezone'; +import omit from 'lodash-es/omit'; export default Route.extend(AuthenticatedRouteMixin, { setupController(controller, model) { controller.set('model', model); - controller.set('dateRanges', this.get('dateRanges')); - controller.set('presentQueryParamValues', this.get('presentQueryParamValues') || {}); - controller.set('allQueryParamValues', this.get('allQueryParamValues') || {}); - controller.set('backendQueryParamValues', this.get('backendQueryParamValues') || {}); + controller.set('dateRanges', this.dateRanges); + controller.set('presentQueryParamValues', this.presentQueryParamValues || {}); + controller.set('allQueryParamValues', this.allQueryParamValues || {}); + controller.set('backendQueryParamValues', this.backendQueryParamValues || {}); - $('ul.nav li').removeClass('active'); - $('ul.nav li.nav-analytics').addClass('active'); + $('ul.navbar-nav li').removeClass('active'); + $('ul.navbar-nav li.nav-analytics').addClass('active'); }, beforeModel() { @@ -67,7 +70,7 @@ export default Route.extend(AuthenticatedRouteMixin, { // 2. So that the default value changes if the user has the app open for // multiple days (we don't want the default value from the very first // load to never be updated again). - let allParams = _.cloneDeep(this.paramsFor(this.routeName) || {}); + let allParams = cloneDeep(this.paramsFor(this.routeName) || {}); if(allParams.date_range) { let range = dateRanges[allParams.date_range]; if(range) { @@ -82,7 +85,7 @@ export default Route.extend(AuthenticatedRouteMixin, { this.set('dateRanges', dateRanges); this.set('allQueryParamValues', allParams); - this.set('backendQueryParamValues', _.omit(allParams, ['date_range'])); + this.set('backendQueryParamValues', omit(allParams, ['date_range'])); }, diff --git a/src/api-umbrella/admin-ui/app/routes/stats/drilldown.js b/src/api-umbrella/admin-ui/app/routes/stats/drilldown.js index 8ee8bf81f..300a521b7 100644 --- a/src/api-umbrella/admin-ui/app/routes/stats/drilldown.js +++ b/src/api-umbrella/admin-ui/app/routes/stats/drilldown.js @@ -24,13 +24,10 @@ export default Base.extend({ prefix: { refreshModel: true, }, - beta_analytics: { - refreshModel: true, - }, }, model() { - let params = this.get('backendQueryParamValues'); + let params = this.backendQueryParamValues; if(this.validateParams(params)) { return StatsDrilldown.find(params); } else { diff --git a/src/api-umbrella/admin-ui/app/routes/stats/logs.js b/src/api-umbrella/admin-ui/app/routes/stats/logs.js index 1d0bc0422..139932ea5 100644 --- a/src/api-umbrella/admin-ui/app/routes/stats/logs.js +++ b/src/api-umbrella/admin-ui/app/routes/stats/logs.js @@ -21,13 +21,10 @@ export default Base.extend({ search: { refreshModel: true, }, - beta_analytics: { - refreshModel: true, - }, }, model() { - let params = this.get('backendQueryParamValues'); + let params = this.backendQueryParamValues; if(this.validateParams(params)) { return StatsLogs.find(params); } else { diff --git a/src/api-umbrella/admin-ui/app/routes/stats/map.js b/src/api-umbrella/admin-ui/app/routes/stats/map.js index 2cbbd920f..357e3b054 100644 --- a/src/api-umbrella/admin-ui/app/routes/stats/map.js +++ b/src/api-umbrella/admin-ui/app/routes/stats/map.js @@ -21,13 +21,10 @@ export default Base.extend({ region: { refreshModel: true, }, - beta_analytics: { - refreshModel: true, - }, }, model() { - let params = this.get('backendQueryParamValues'); + let params = this.backendQueryParamValues; if(this.validateParams(params)) { return StatsMap.find(params); } else { diff --git a/src/api-umbrella/admin-ui/app/routes/stats/users.js b/src/api-umbrella/admin-ui/app/routes/stats/users.js index e510e3c47..b363f27d7 100644 --- a/src/api-umbrella/admin-ui/app/routes/stats/users.js +++ b/src/api-umbrella/admin-ui/app/routes/stats/users.js @@ -17,9 +17,6 @@ export default Base.extend({ search: { refreshModel: true, }, - beta_analytics: { - refreshModel: true, - }, }, model() { diff --git a/src/api-umbrella/admin-ui/app/routes/website-backends/base.js b/src/api-umbrella/admin-ui/app/routes/website-backends/base.js index ebe15b271..b335cb0b1 100644 --- a/src/api-umbrella/admin-ui/app/routes/website-backends/base.js +++ b/src/api-umbrella/admin-ui/app/routes/website-backends/base.js @@ -6,7 +6,7 @@ export default Route.extend(AuthenticatedRouteMixin, { setupController(controller, model) { controller.set('model', model); - $('ul.nav li').removeClass('active'); - $('ul.nav li.nav-config').addClass('active'); + $('ul.navbar-nav li').removeClass('active'); + $('ul.navbar-nav li.nav-config').addClass('active'); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/website-backends/edit.js b/src/api-umbrella/admin-ui/app/routes/website-backends/edit.js index 5729ecfb5..a92d7fbe9 100644 --- a/src/api-umbrella/admin-ui/app/routes/website-backends/edit.js +++ b/src/api-umbrella/admin-ui/app/routes/website-backends/edit.js @@ -3,6 +3,6 @@ import Form from './form'; export default Form.extend({ model(params) { this.clearStoreCache(); - return this.get('store').findRecord('website-backend', params.website_backend_id, { reload: true }); + return this.store.findRecord('website-backend', params.website_backend_id, { reload: true }); }, }); diff --git a/src/api-umbrella/admin-ui/app/routes/website-backends/new.js b/src/api-umbrella/admin-ui/app/routes/website-backends/new.js index 56682bb55..60a83e7d8 100644 --- a/src/api-umbrella/admin-ui/app/routes/website-backends/new.js +++ b/src/api-umbrella/admin-ui/app/routes/website-backends/new.js @@ -3,7 +3,7 @@ import Form from './form'; export default Form.extend({ model() { this.clearStoreCache(); - return this.get('store').createRecord('website-backend', { + return this.store.createRecord('website-backend', { serverPort: 80, }); }, diff --git a/src/api-umbrella/admin-ui/app/services/busy.js b/src/api-umbrella/admin-ui/app/services/busy.js new file mode 100644 index 000000000..80b4063d3 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/services/busy.js @@ -0,0 +1,12 @@ +import Evented from '@ember/object/evented'; +import Service from '@ember/service'; + +export default Service.extend(Evented, { + hide() { + this.trigger('hide'); + }, + + show(options) { + this.trigger('show', options); + }, +}); diff --git a/src/api-umbrella/admin-ui/app/styles/_base.scss b/src/api-umbrella/admin-ui/app/styles/_base.scss index 6d0f303b0..6415a15cb 100644 --- a/src/api-umbrella/admin-ui/app/styles/_base.scss +++ b/src/api-umbrella/admin-ui/app/styles/_base.scss @@ -1,5 +1,13 @@ body { - padding-top: 60px; + padding-top: 66px; +} + +.code-block, +.tippy-tooltip pre { + background-color: $light; + border: $card-border-width solid $card-border-color; + @include border-radius($card-border-radius); + padding: 0.5rem; } .navbar .navbar-brand { @@ -18,192 +26,17 @@ body { z-index: 999; } -#results_table .table th, -#results_table .table td { - white-space: nowrap; -} - -#results_table .table thead th { - color: $link-color; -} - -table.table thead .sorting, -table.table thead .sorting_asc, -table.table thead .sorting_desc { - background: none; -} - -table.table thead .sorting:after, -table.table thead .sorting_asc:after, -table.table thead .sorting_desc:after { - opacity: 1.0; - font-family: FontAwesome; - color: $gray-lighter; - margin-left: 8px; -} - -table.table thead .sorting_asc:after, -table.table thead .sorting_desc:after { - color: $link-color; -} - -table.table thead .sorting:after { - content: "\F0DC"; -} - -table.table thead .sorting_asc:after { - content: "\F0DE"; -} - -table.table thead .sorting_desc:after { - content: "\F0DD"; -} - -div.dataTables_info { - padding: 0px; - font-size: 13px; - color: $text-color-light; -} - -div.dataTables_paginate { - text-align: center !important; - float: none; -} - -div.dataTables_length { - text-align: right; - - label { - float: none; - font-size: 13px; - color: $text-color-light; - } - - select { - font-size: 13px; - } -} - -div.dataTables_wrapper { - position: relative; - - div.dataTables_filter { - label { - float: left; - } - - input { - margin-left: 0px; - } - } -} - -.dataTables_processing { - position: absolute; - top: 50%; - left: 50%; - width: 250px; - height: 60px; - margin-left: -125px; - margin-top: -30px; - border: 1px solid #ddd; - text-align: center; - color: $gray-light; - font-size: 32px; - line-height: 60px; - background-color: white; -} - -.form-horizontal.condensed .form-group { - margin-bottom: 5px; -} - -fieldset { - margin: 12px 0px 32px 0px; - padding: 6px 0px 6px 12px; -} - -fieldset.collapsible { - margin-bottom: 12px; - - .collapse.in { - margin-bottom: 24px; - } -} - -legend { - margin-bottom: 2px; - border-bottom-width: 1px; - border-color: $gray-light; - margin-left: -12px; - padding: 6px 10px 3px 2px; - line-height: 22px; - background-color: #fff; -} - -legend a { - display: block; -} - -.fieldset-note { - font-size: 85%; - color: $text-color-light; - margin-bottom: 4px; -} - -fieldset .table { - margin-bottom: 4px; -} - -.control-group.error .help-block { - margin-top: -8px; - font-style: italic; -} - -.form-horizontal .control-group.error .help-block { - margin-top: 0px; -} - -.error-messages { - h4 { - margin-bottom: 4px; - } - - ul { - margin-bottom: 0px; - - ul { - margin-top: 10px; - margin-bottom: 10px; - } - } - - li { - p { - margin: 10px 0px 0px 0px; - } - - p:first-child { - margin: 0px; - } - } -} - -.error-page { - margin: 0px auto; - max-width: 420px; -} - .arrow { margin-top: 24px; text-align: center; - color: $gray-light; - font-size: 10px; - line-height: 10px; -} + color: $gray-600; + font-size: 12px; + line-height: 12px; -.arrow i { - color: lighten($gray-light, 16%); + .svg-inline--fa { + color: lighten($gray-600, 16%); + vertical-align: middle; + } } .arrow-vertical { @@ -214,54 +47,10 @@ fieldset .table { vertical-align: middle; } -a { - .fa { - text-decoration: none; - margin-right: 3px; - } -} - -.fa-space-right { - margin-right: 3px; -} - -.table .table-row-actions { - text-align: right; - white-space: nowrap; - - a { - margin-left: 24px; - } -} - a.remove-action { font-size: 12px; - color: $brand-danger; -} - -.table-actions { -} - -.form-extra-actions { - margin-top: 20px; -} - -table input { - margin-bottom: 0px !important; -} - -legend a:hover { - text-decoration: none; -} - -a[data-toggle='collapse'] .fa-caret-down:before { - display: block; - width: 16px; - text-align: center; -} - -a[data-toggle='collapse'].collapsed .fa-caret-down:before { - content: "\f0da"; + vertical-align: middle; + color: $danger; } a:hover { @@ -276,60 +65,7 @@ a:hover { text-align: right; font-size: 11px; line-height: 13px; - color: $text-color-lighter; -} - -th.reorder-handle, -td.reorder-handle { - display: none; - width: 60px; - text-align: center; - cursor: move; -} - -.reorder-active { - th.reorder-handle, - td.reorder-handle { - display: table-cell; - } -} - -.reorder-placeholder th, -.reorder-placeholder td { - background-color: $state-warning-bg !important; -} - -.form-horizontal .ace_editor { - height: 96px; - border: 1px solid $input-border; - border-radius: $input-border-radius; -} - -.control-label a[rel='tooltip'] { - margin-left: 8px; -} - -.qtip-bootstrap { - font-size: 12px; - max-width: 320px; - - pre { - font-size: 11px; - line-height: 15px; - } -} - -.qtip-wide { - max-width: 700px; -} - -.qtip-forced-wide { - min-width: 400px; - max-width: 600px; -} - -.qtip-content p:last-child { - margin-bottom: 0px; + color: $text-muted; } .query-syntax-help .example { @@ -341,28 +77,6 @@ td.reorder-handle { font-size: 11px; } -.table-small { - font-size: 13px; -} - -.table tr.empty td { - text-align: center; - font-size: 12px; - font-style: italic; - color: $text-color-light; -} - -.table th.text-center, -.table td.text-center { - text-align: center; -} - -.table tr.line-bottom th, -.table tr.line-bottom td { - border-top: none; - border-bottom: 1px solid $table-border-color; -} - .diff { del { text-decoration: none; @@ -395,13 +109,13 @@ td.reorder-handle { } #version_footer .row-fluid { - border-top: 1px solid $gray-lighter; + border-top: 1px solid $gray-200; text-align: center; margin-top: 3em; margin-bottom: 1em; padding-top: 1em; font-size: 11px; - color: $text-color-lighter; + color: $text-muted; } // For when we want to submit a form with the enter key, but don't want a real @@ -418,8 +132,6 @@ td.reorder-handle { margin: 0px; } -table.table-max-width-cells td { - max-width: 400px; - text-overflow: ellipsis; - overflow: hidden; +.btn-light { + @include button-variant($gray-300, $gray-300); } diff --git a/src/api-umbrella/admin-ui/app/styles/_busy-blocker.scss b/src/api-umbrella/admin-ui/app/styles/_busy-blocker.scss index d66011304..1910d1d8b 100644 --- a/src/api-umbrella/admin-ui/app/styles/_busy-blocker.scss +++ b/src/api-umbrella/admin-ui/app/styles/_busy-blocker.scss @@ -46,7 +46,7 @@ position: fixed; top: 0; width: 100%; - z-index: $zindex-modal-background; + z-index: $zindex-modal-backdrop; } .busy-blocker__content { diff --git a/src/api-umbrella/admin-ui/app/styles/_codemirror.scss b/src/api-umbrella/admin-ui/app/styles/_codemirror.scss new file mode 100644 index 000000000..55f0c2c40 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/styles/_codemirror.scss @@ -0,0 +1,7 @@ +@import "node_modules/codemirror/lib/codemirror"; + +.CodeMirror { + @extend .form-control; + padding: 0px; + height: 8rem; +} diff --git a/src/api-umbrella/admin-ui/app/styles/_forms.scss b/src/api-umbrella/admin-ui/app/styles/_forms.scss new file mode 100644 index 000000000..a68d65029 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/styles/_forms.scss @@ -0,0 +1,144 @@ +label { + font-weight: bold; + margin-bottom: 0.1rem; +} + +.custom-control-label, +.form-check-label { + font-weight: normal; +} + +.form-horizontal .col-form-label { + text-align: right; +} + +.form-fields-checkboxes-field .form-horizontal .col-form-label { + padding-top: 0px; +} + +.form-horizontal.condensed .form-group { + margin-bottom: 5px; +} + +fieldset { + margin: 12px 0px 32px 0px; + padding: 6px 0px; +} + +fieldset.collapsible { + margin-bottom: 12px; + + .collapse.in { + margin-bottom: 24px; + } +} + +legend { + margin-bottom: 2px; + border-width: 0px 0px 1px 0px; + border-style: solid; + border-color: $gray-600; + padding: 6px 10px 3px 2px; + line-height: 22px; + background-color: #fff; +} + +legend a { + display: block; + color: $link-color !important; +} + +legend a:hover { + color: $link-hover-color !important; + text-decoration: none; +} + +.fieldset-note { + font-size: 85%; + color: $text-muted; + margin-bottom: 4px; +} + +fieldset .table { + margin-bottom: 4px; +} + +.has-error { + .form-control { + @extend .is-invalid; + } + + .invalid-feedback, + .invalid-tooltip { + display: block; + } +} + +.error-messages { + h4 { + margin-bottom: 4px; + } + + ul { + margin-bottom: 0px; + + ul { + margin-top: 10px; + margin-bottom: 10px; + } + } + + li { + p { + margin: 10px 0px 0px 0px; + } + + p:first-child { + margin: 0px; + } + } +} + +.error-page { + margin: 0px auto; + max-width: 420px; +} + +.form-extra-actions { + margin-top: 20px; +} + +.custom-control.custom-control-no-label { + display: inline-block; + padding-left: 1rem; + + .custom-control-label { + display: block; + width: 0px; + } + + .custom-control-label::before, + .custom-control-label::after { + left: -1rem; + } +} + +.btn { + .btn-loading-label { + display: none; + + .svg-inline--fa:first-child { + @extend .mr-2; + } + } + + &.btn-loading { + .btn-label { + display: none; + } + + .btn-loading-label { + display: inline; + } + } +} diff --git a/src/api-umbrella/admin-ui/app/styles/_icons.scss b/src/api-umbrella/admin-ui/app/styles/_icons.scss new file mode 100644 index 000000000..9e1306bbd --- /dev/null +++ b/src/api-umbrella/admin-ui/app/styles/_icons.scss @@ -0,0 +1,7 @@ +a .svg-inline--fa:first-child { + @extend .mr-1; +} + +a[data-toggle='collapse'].collapsed .fa-caret-down { + transform: rotate(270deg); +} diff --git a/src/api-umbrella/admin-ui/app/styles/_input-xs.scss b/src/api-umbrella/admin-ui/app/styles/_input-xs.scss deleted file mode 100644 index 0b19e4dab..000000000 --- a/src/api-umbrella/admin-ui/app/styles/_input-xs.scss +++ /dev/null @@ -1,5 +0,0 @@ -$padding-xs-vertical: 3px; -$padding-xs-horizontal: 5px; -$input-height-xs: (floor($font-size-small * $line-height-small) + ($padding-xs-vertical * 2) + 2) !default; - -@include input-size('.input-xs', $input-height-xs, $padding-xs-vertical, $padding-xs-horizontal, $font-size-small, $line-height-small, $input-border-radius-small); diff --git a/src/api-umbrella/admin-ui/app/styles/_query-builder.scss b/src/api-umbrella/admin-ui/app/styles/_query-builder.scss new file mode 100644 index 000000000..86fc47d47 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/styles/_query-builder.scss @@ -0,0 +1,56 @@ +@import "node_modules/jQuery-QueryBuilder/dist/scss/default.scss"; + +.query-builder { + .pull-right { + float: right !important; + } + + .rules-group-container { + width: 100%; + background: #edf5fc; + background: rgba(237, 245, 252, 0.5); + border-color: #96bedc; + margin-top: 0px; + + .btn-xs { + @extend .btn-sm; + } + } + + .rule-container { + display: flex; + + .rule-header { + @extend .order-last; + margin-left: auto; + } + + .rule-value-container { + @extend .flex-fill; + border-width: 0px; + padding-left: 0px; + padding-right: 16px; + + input[type='number'].form-control, + input[type='text'].form-control, + select.form-control { + width: 100%; + margin-right: 16px; + } + + input[type='number'].form-control { + max-width: 6rem; + } + } + + select.form-control { + @extend .custom-select; + @extend .custom-select-sm; + } + + input[type='number'].form-control, + input[type='text'].form-control { + @extend .form-control-sm; + } + } +} diff --git a/src/api-umbrella/admin-ui/app/styles/_selectize.scss b/src/api-umbrella/admin-ui/app/styles/_selectize.scss index 709e0fcfb..958807f72 100644 --- a/src/api-umbrella/admin-ui/app/styles/_selectize.scss +++ b/src/api-umbrella/admin-ui/app/styles/_selectize.scss @@ -1,3 +1,5 @@ +@import "node_modules/selectize/dist/css/selectize.default"; + .selectize-dropdown { z-index: $zindex-modal + 10; } diff --git a/src/api-umbrella/admin-ui/app/styles/_stats.scss b/src/api-umbrella/admin-ui/app/styles/_stats.scss index e3eb7dd0d..663a053ea 100644 --- a/src/api-umbrella/admin-ui/app/styles/_stats.scss +++ b/src/api-umbrella/admin-ui/app/styles/_stats.scss @@ -35,20 +35,12 @@ margin: 0px auto 20px auto; } -.advanced-filter { - .help-block { - font-size: 85%; - line-height: 110%; - margin-top: 2px; - } -} - .filter-times { text-align: right; + margin-bottom: 8px; #interval_buttons { - display: inline-block; - margin: 4px 30px 0px 0px; + margin-right: 30px; } #reportrange { @@ -88,11 +80,6 @@ #filters_ui { max-width: 960px; - .filter-type-toggle { - font-size: 12px; - text-align: right; - } - #search_field { .input-append { display: block; @@ -114,30 +101,3 @@ } } } - -.query-builder { - .btn-xs { - font-size: 12px; - padding: 1px 5px; - } - - .rule-container input[type=number], - .rule-container input[type=text], - .rule-container select { - font-size: 11px; - height: 24px; - margin: 0px; - box-sizing: border-box; - padding: 1px 3px; - } - - .rule-container input[type=number] { - width: 102px; - } - - .rules-group-container { - background: #edf5fc; - background: rgba(237, 245, 252, 0.5); - border-color: #96bedc; - } -} diff --git a/src/api-umbrella/admin-ui/app/styles/_tables.scss b/src/api-umbrella/admin-ui/app/styles/_tables.scss new file mode 100644 index 000000000..b481b780f --- /dev/null +++ b/src/api-umbrella/admin-ui/app/styles/_tables.scss @@ -0,0 +1,187 @@ +@import "node_modules/datatables.net-bs4/css/dataTables.bootstrap4"; + +.table-nowrap th, +.table-nowrap td { + white-space: nowrap; +} + +table.dataTable.table thead .sorting:before, +table.dataTable.table thead .sorting:after, +table.dataTable.table thead .sorting_asc:before, +table.dataTable.table thead .sorting_asc:after, +table.dataTable.table thead .sorting_desc:before, +table.dataTable.table thead .sorting_desc:after, { + display: none; +} + +table.dataTable.table thead .sorting, +table.dataTable.table thead .sorting_asc, +table.dataTable.table thead .sorting_desc { + .svg-inline--fa { + color: $gray-500; + position: absolute; + right: 0.5rem; + top: 0.5rem; + } +} + +table.dataTable.table thead .sorting_asc, +table.dataTable.table thead .sorting_desc { + .svg-inline--fa { + color: $link-color; + } +} + +table.dataTable.table thead .sorting_disabled { + .svg-inline--fa { + display: none; + } +} + +table.dataTable.table thead .sorting { + .fa-sort-up, + .fa-sort-down { + display: none; + } +} + +table.dataTable.table thead .sorting_asc { + .fa-sort, + .fa-sort-down { + display: none; + } +} + +table.dataTable.table thead .sorting_desc { + .fa-sort, + .fa-sort-up { + display: none; + } +} + +table.dataTable.table-sm > thead > tr > th { + padding-right: 1.5rem; +} + +div.dataTables_wrapper div.dataTables_paginate ul.pagination { + justify-content: center; +} + +div.dataTables_info { + padding: 0px; + font-size: 13px; + //color: $text-color-light; +} + +div.dataTables_paginate { + text-align: center !important; + float: none; +} + +div.dataTables_length { + text-align: right; + + label { + float: none; + font-size: 13px; + //color: $text-color-light; + } + + select { + font-size: 13px; + } +} + +div.dataTables_wrapper { + position: relative; + + div.dataTables_filter { + label { + float: left; + margin-bottom: .4rem; + } + + input { + margin-left: 0px; + } + } +} + +.dataTables_processing { + position: absolute; + top: 50%; + left: 50%; + width: 250px; + height: 60px; + margin-left: -125px; + margin-top: -30px; + border: 1px solid #ddd; + text-align: center; + color: $gray-600; + font-size: 32px; + line-height: 60px; + background-color: white; +} + +.table .table-row-actions { + text-align: right; + white-space: nowrap; + vertical-align: middle; + + a { + margin-left: 24px; + vertical-align: middle; + } +} + +table input { + margin-bottom: 0px !important; +} + +th.reorder-handle, +td.reorder-handle { + display: none; + width: 60px; + text-align: center; + cursor: move; +} + +.reorder-active { + th.reorder-handle, + td.reorder-handle { + display: table-cell; + } +} + +.reorder-placeholder th, +.reorder-placeholder td { + background-color: $yellow !important; +} + +.table-small { + font-size: 13px; +} + +.table tr.empty td { + text-align: center; + font-size: 12px; + font-style: italic; + color: $text-muted; +} + +.table th.text-center, +.table td.text-center { + text-align: center; +} + +.table tr.line-bottom th, +.table tr.line-bottom td { + border-top: none; + border-bottom: 1px solid $table-border-color; +} + +table.table-max-width-cells td { + max-width: 400px; + text-overflow: ellipsis; + overflow: hidden; +} diff --git a/src/api-umbrella/admin-ui/app/styles/_tooltips.scss b/src/api-umbrella/admin-ui/app/styles/_tooltips.scss new file mode 100644 index 000000000..e1560b5e0 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/styles/_tooltips.scss @@ -0,0 +1,31 @@ +@import "node_modules/tippy.js/index"; +@import "node_modules/tippy.js/themes/light-border"; + +.btn-link.btn-tooltip { + padding: 0px; + margin: 0px; + border-width: 0px; + vertical-align: baseline; + font-size: inherit; +} + +.col-form-label .btn-link.btn-tooltip { + margin-left: 8px; +} + +.tippy-tooltip { + text-align: left; + + p:last-child { + margin-bottom: 0px; + } +} + +.tippy-tooltip.wide-theme { + max-width: 700px; +} + +.tippy-tooltip.forced-wide-theme { + min-width: 400px; + max-width: 600px; +} diff --git a/src/api-umbrella/admin-ui/app/styles/_variables.scss b/src/api-umbrella/admin-ui/app/styles/_variables.scss new file mode 100644 index 000000000..3dbaa4a23 --- /dev/null +++ b/src/api-umbrella/admin-ui/app/styles/_variables.scss @@ -0,0 +1,6 @@ +$enable-shadows: true; + +// Make placeholder text lighter, or else it's hard to distinguish it between +// text that's filled in: https://github.com/twbs/bootstrap/issues/21624 This +// new color is based on the browser defaults. +$input-placeholder-color: #a9a9a9; diff --git a/src/api-umbrella/admin-ui/app/styles/_variables_post.scss b/src/api-umbrella/admin-ui/app/styles/_variables_post.scss deleted file mode 100644 index 8d8fe42bf..000000000 --- a/src/api-umbrella/admin-ui/app/styles/_variables_post.scss +++ /dev/null @@ -1,10 +0,0 @@ -$headings-color: $text-color; - -$navbar-default-bg: lighten($gray-base, 23%); -$navbar-default-link-hover-bg: darken($navbar-default-bg, 10%); -$navbar-default-link-active-bg: $navbar-default-link-hover-bg; - -$gray-lighter: lighten($gray-base, 85%); - -$text-color-light: lighten($text-color, 16%); -$text-color-lighter: lighten($text-color, 28%); diff --git a/src/api-umbrella/admin-ui/app/styles/_variables_pre.scss b/src/api-umbrella/admin-ui/app/styles/_variables_pre.scss deleted file mode 100644 index 03dfeffad..000000000 --- a/src/api-umbrella/admin-ui/app/styles/_variables_pre.scss +++ /dev/null @@ -1,7 +0,0 @@ -$text-color: #313131; - -// Use the default bootstrap padding & heights, rather than the larger defaults -// from the Cerulean theme. -$padding-base-vertical: 6px; -$padding-large-vertical: 10px; -$modal-inner-padding: 15px; diff --git a/src/api-umbrella/admin-ui/app/styles/app.scss b/src/api-umbrella/admin-ui/app/styles/app.scss index 7519dd740..1b2b32139 100644 --- a/src/api-umbrella/admin-ui/app/styles/app.scss +++ b/src/api-umbrella/admin-ui/app/styles/app.scss @@ -1,18 +1,15 @@ -@import "_variables_pre.scss"; -@import "node_modules/bootswatch/cerulean/_variables.scss"; -@import "_variables_post.scss"; -@import "node_modules/bootstrap-sass/assets/stylesheets/_bootstrap.scss"; -@import "node_modules/bootswatch/cerulean/_bootswatch.scss"; -@import "node_modules/font-awesome/scss/font-awesome.scss"; +@import "_variables.scss"; +@import "node_modules/bootstrap/scss/bootstrap.scss"; +@import "_tooltips"; -@import "node_modules/jQuery-QueryBuilder/dist/scss/default.scss"; -@import "node_modules/bootstrap-daterangepicker/daterangepicker.scss"; -@import "node_modules/pnotify/dist/pnotify"; -@import "node_modules/pnotify/dist/pnotify.buttons"; -@import "node_modules/pnotify/dist/pnotify.mobile"; +@import "_query-builder.scss"; +@import "node_modules/daterangepicker/daterangepicker"; @import "_noscript.scss"; @import "_busy-blocker.scss"; -@import "_input-xs.scss"; @import "_base.scss"; +@import "_forms.scss"; +@import "_codemirror.scss"; +@import "_icons.scss"; +@import "_tables.scss"; @import "_stats.scss"; @import "_selectize.scss"; diff --git a/src/api-umbrella/admin-ui/app/templates/admin-groups/index.hbs b/src/api-umbrella/admin-ui/app/templates/admin-groups/index.hbs index 6131895ed..20508a620 100644 --- a/src/api-umbrella/admin-ui/app/templates/admin-groups/index.hbs +++ b/src/api-umbrella/admin-ui/app/templates/admin-groups/index.hbs @@ -1,7 +1,7 @@

Admin Groups

- {{#link-to "admin_groups.new" class="btn btn-primary"}} Add New Admin Group{{/link-to}} + {{#link-to "admin_groups.new" class="btn btn-primary"}}{{fa-icon "plus"}} Add New Admin Group{{/link-to}}
{{admin-groups/index-table}} diff --git a/src/api-umbrella/admin-ui/app/templates/admins/index.hbs b/src/api-umbrella/admin-ui/app/templates/admins/index.hbs index 029a9e433..0be1d02b5 100644 --- a/src/api-umbrella/admin-ui/app/templates/admins/index.hbs +++ b/src/api-umbrella/admin-ui/app/templates/admins/index.hbs @@ -1,7 +1,7 @@

Admins

- {{#link-to "admins.new" class="btn btn-primary"}} Add New Admin{{/link-to}} + {{#link-to "admins.new" class="btn btn-primary"}}{{fa-icon "plus"}} Add New Admin{{/link-to}}
{{admins/index-table}} diff --git a/src/api-umbrella/admin-ui/app/templates/api-scopes/index.hbs b/src/api-umbrella/admin-ui/app/templates/api-scopes/index.hbs index 1fc8e3740..eec12ee3c 100644 --- a/src/api-umbrella/admin-ui/app/templates/api-scopes/index.hbs +++ b/src/api-umbrella/admin-ui/app/templates/api-scopes/index.hbs @@ -1,7 +1,7 @@

API Scopes

- {{#link-to "api_scopes.new" class="btn btn-primary"}} Add New API Scope{{/link-to}} + {{#link-to "api_scopes.new" class="btn btn-primary"}}{{fa-icon "plus"}} Add New API Scope{{/link-to}}
{{api-scopes/index-table}} diff --git a/src/api-umbrella/admin-ui/app/templates/api-users/index.hbs b/src/api-umbrella/admin-ui/app/templates/api-users/index.hbs index cead37668..1a64f94d1 100644 --- a/src/api-umbrella/admin-ui/app/templates/api-users/index.hbs +++ b/src/api-umbrella/admin-ui/app/templates/api-users/index.hbs @@ -1,7 +1,7 @@

API Users

- {{#link-to "api_users.new" class="btn btn-primary"}} Add New API User{{/link-to}} + {{#link-to "api_users.new" class="btn btn-primary"}}{{fa-icon "plus"}} Add New API User{{/link-to}}
{{api-users/index-table}} diff --git a/src/api-umbrella/admin-ui/app/templates/apis/index.hbs b/src/api-umbrella/admin-ui/app/templates/apis/index.hbs index a2bae09cd..58f01b0b3 100644 --- a/src/api-umbrella/admin-ui/app/templates/apis/index.hbs +++ b/src/api-umbrella/admin-ui/app/templates/apis/index.hbs @@ -1,7 +1,7 @@

API Backends

- {{#link-to "apis.new" class="btn btn-primary"}} Add API Backend{{/link-to}} + {{#link-to "apis.new" class="btn btn-primary"}}{{fa-icon "plus"}} Add API Backend{{/link-to}}
{{apis/index-table}} diff --git a/src/api-umbrella/admin-ui/app/templates/application.hbs b/src/api-umbrella/admin-ui/app/templates/application.hbs index b51bdc7c6..306274890 100644 --- a/src/api-umbrella/admin-ui/app/templates/application.hbs +++ b/src/api-umbrella/admin-ui/app/templates/application.hbs @@ -1,70 +1,63 @@ {{#if session.isAuthenticated}} - diff --git a/src/api-umbrella/admin-ui/app/templates/components/admin-groups/index-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/admin-groups/index-table.hbs index 52f6a684f..ad17bee6b 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/admin-groups/index-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/admin-groups/index-table.hbs @@ -1,3 +1,3 @@ -
-
+
+
diff --git a/src/api-umbrella/admin-ui/app/templates/components/admin-groups/record-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/admin-groups/record-form.hbs index f49739575..595f10861 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/admin-groups/record-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/admin-groups/record-form.hbs @@ -24,7 +24,7 @@
- +
{{#if model.id}} @@ -35,7 +35,7 @@
{{#if model.id}}
- Delete Admin Group + {{fa-icon "times"}}Delete Admin Group
{{/if}} {{/fields-for}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/admins/index-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/admins/index-table.hbs index e05f53a67..12fd4e14b 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/admins/index-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/admins/index-table.hbs @@ -1,7 +1,7 @@ -
-
+
+
- Download CSV + Download CSV
diff --git a/src/api-umbrella/admin-ui/app/templates/components/admins/record-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/admins/record-form.hbs index c2e7da678..c258331fb 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/admins/record-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/admins/record-form.hbs @@ -5,32 +5,45 @@
User Info - {{f.text-field "username" label=(t "mongoid.attributes.admin.username")}} - {{#unless session.data.authenticated.username_is_email}} - {{f.text-field "email" label=(t "mongoid.attributes.admin.email")}} - {{/unless}} + {{#if currentAdmin.permissions.admin_manage}} + {{f.text-field "username" label=(t "mongoid.attributes.admin.username")}} + {{#unless session.data.authenticated.username_is_email}} + {{f.text-field "email" label=(t "mongoid.attributes.admin.email")}} + {{/unless}} + {{else}} + {{f.static-field "username" label=(t "mongoid.attributes.admin.username")}} + {{#unless session.data.authenticated.username_is_email}} + {{f.static-field "email" label=(t "mongoid.attributes.admin.email")}} + {{/unless}} + {{/if}} {{#if model.name}} {{f.static-field "name" label=(t "mongoid.attributes.admin.name")}} {{/if}} - {{f.textarea-field "notes" label=(t "mongoid.attributes.admin.notes")}} - {{#if model.id}} - {{#unless model.currentSignInAt}} - {{f.checkbox-field "sendInviteEmail" label="Resend invite email"}} - {{/unless}} + {{#if currentAdmin.permissions.admin_manage}} + {{f.textarea-field "notes" label=(t "mongoid.attributes.admin.notes")}} + {{#if model.id}} + {{#unless model.currentSignInAt}} + {{f.checkbox-field "sendInviteEmail" label="Resend invite email"}} + {{/unless}} + {{else}} + {{f.checkbox-field "sendInviteEmail" label="Send invite email"}} + {{/if}} {{else}} - {{f.checkbox-field "sendInviteEmail" label="Send invite email"}} + {{f.static-field "notes" label=(t "mongoid.attributes.admin.notes")}} {{/if}}
{{#if model.authenticationToken}} {{#if session.data.authenticated.local_auth_enabled}} -
- {{t "devise.passwords.edit.change_your_password"}} + {{#if currentAdmin.permissions.admin_manage}} +
+ {{t "devise.passwords.edit.change_your_password"}} - {{f.password-field "currentPassword" label=(t "mongoid.attributes.admin.current_password")}} - {{f.password-field "password" label=(t "devise.passwords.edit.new_password") hint=(t "devise.passwords.password_length_hint" min=session.data.authenticated.password_length_min)}} - {{f.password-field "passwordConfirmation" label=(t "devise.passwords.edit.confirm_new_password")}} -
+ {{f.password-field "currentPassword" label=(t "mongoid.attributes.admin.current_password")}} + {{f.password-field "password" label=(t "devise.passwords.edit.new_password") hint=(t "devise.passwords.password_length_hint" min=session.data.authenticated.password_length_min)}} + {{f.password-field "passwordConfirmation" label=(t "devise.passwords.edit.confirm_new_password")}} +
+ {{/if}} {{/if}}
@@ -42,18 +55,22 @@
{{/if}} -
- Permissions + {{#if currentAdmin.permissions.admin_manage}} +
+ Permissions - {{f.checkboxes-field "groupIds" label=(t "mongoid.attributes.admin.groups") options=groupOptions}} - {{#if currentAdmin.superuser}} - {{f.checkbox-field "superuser" label=(t "mongoid.attributes.admin.superuser")}} - {{/if}} -
+ {{f.checkboxes-field "groupIds" label=(t "mongoid.attributes.admin.groups") options=groupOptions}} + {{#if currentAdmin.superuser}} + {{f.checkbox-field "superuser" label=(t "mongoid.attributes.admin.superuser")}} + {{/if}} +
+ {{/if}}
- + {{#if currentAdmin.permissions.admin_manage}} + + {{/if}}
{{#if model.id}} @@ -64,10 +81,12 @@ {{/if}}
- {{#if model.id}} -
- Delete Admin -
+ {{#if currentAdmin.permissions.admin_manage}} + {{#if model.id}} +
+ {{fa-icon "times"}}Delete Admin +
+ {{/if}} {{/if}} {{/fields-for}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/api-scopes/index-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/api-scopes/index-table.hbs index 52f6a684f..ad17bee6b 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/api-scopes/index-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/api-scopes/index-table.hbs @@ -1,3 +1,3 @@ -
-
+
+
diff --git a/src/api-umbrella/admin-ui/app/templates/components/api-scopes/record-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/api-scopes/record-form.hbs index 568266bf6..2894f1723 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/api-scopes/record-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/api-scopes/record-form.hbs @@ -10,7 +10,7 @@
- +
{{#if model.id}} @@ -21,7 +21,7 @@
{{#if model.id}}
- Delete API Scope + {{fa-icon "times"}}Delete API Scope
{{/if}} {{/fields-for}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/api-users/index-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/api-users/index-table.hbs index 52f6a684f..ad17bee6b 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/api-users/index-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/api-users/index-table.hbs @@ -1,3 +1,3 @@ -
-
+
+
diff --git a/src/api-umbrella/admin-ui/app/templates/components/api-users/record-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/api-users/record-form.hbs index 626bb3667..919b8ed30 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/api-users/record-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/api-users/record-form.hbs @@ -16,7 +16,7 @@ {{/f.static-field}} {{#f.static-field "apiKey" label="API Key"}} {{#if model.apiKey}} - {{model.apiKeyPreview}} {{t "admin.reveal_action"}} + {{model.apiKeyPreview}} {{t "admin.reveal_action"}} {{else}} {{model.apiKeyPreview}} {{/if}} @@ -46,23 +46,23 @@
- +
Created: {{format-date model.createdAt}}{{#if model.creator}} by {{model.creator.username}}{{/if}}
Last Updated: {{format-date model.updatedAt}}{{#if model.updater}} by {{model.updater.username}}{{/if}}
E-mail Verified: {{model.emailVerified}}
{{#if model.registrationIp}} - Registration IP: {{model.registrationIp}}
+ Registration IP: {{model.registrationIp}}
{{/if}} {{#if model.registrationUserAgent}} - Registration User Agent: {{model.registrationUserAgent}}
+ Registration User Agent: {{model.registrationUserAgent}}
{{/if}} {{#if model.registrationReferer}} - Registration Referer: {{model.registrationReferer}}
+ Registration Referer: {{model.registrationReferer}}
{{/if}} {{#if model.registrationOrigin}} - Registration Origin: {{model.registrationOrigin}}
+ Registration Origin: {{model.registrationOrigin}}
{{/if}}
diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/index-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/index-table.hbs index 65a3eecc8..3c8f03af6 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/index-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/index-table.hbs @@ -1,6 +1,6 @@ -
-
+
+
- +
diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/record-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/record-form.hbs index b84445615..d6c8690c6 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/record-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/record-form.hbs @@ -20,7 +20,7 @@ {{f.text-field "frontendHost" label="Frontend Host"}}
- + {{fa-icon "arrow-right" size="2x"}}
rewrite to
@@ -37,7 +37,7 @@
- +
{{#fields-for model=model.settings style="horizontal-wide-labels" as |f|}} {{f.text-field "appendQueryString" label="Append Query String Parameters" placeholder="param1=value¶m2=value"}} @@ -52,7 +52,7 @@
- +

{{t "admin.api.sub_settings.note"}}

{{apis/sub-settings-table model=model roleOptions=roleOptions}} @@ -60,7 +60,7 @@
- +

{{t "admin.api.rewrites.note"}}

{{apis/rewrite-table model=model}} @@ -68,35 +68,35 @@
- +
{{#fields-for model=model style="horizontal-wide-labels" as |f|}} {{f.select-field "balanceAlgorithm" label="Balance Algorithm" options=balanceAlgorithmOptions}} - {{/fields-for}} + {{/fields-for}} {{#fields-for model=model.settings.errorTemplates style="horizontal-wide-labels" as |f|}}

{{t "mongoid.attributes.api/settings.error_templates"}}

- {{f.ace-field "json" label=(t "admin.api.settings.error_templates_json") tooltip=(t "admin.api.settings.error_templates_json_tooltip_markdown") mode="json"}} - {{f.ace-field "xml" label=(t "admin.api.settings.error_templates_xml") tooltip=(t "admin.api.settings.error_templates_xml_tooltip_markdown") mode="xml"}} - {{f.ace-field "csv" label=(t "admin.api.settings.error_templates_csv") tooltip=(t "admin.api.settings.error_templates_csv_tooltip_markdown") mode="text"}} + {{f.codemirror-field "json" label=(t "admin.api.settings.error_templates_json") tooltip=(t "admin.api.settings.error_templates_json_tooltip_markdown") mode="application/json"}} + {{f.codemirror-field "xml" label=(t "admin.api.settings.error_templates_xml") tooltip=(t "admin.api.settings.error_templates_xml_tooltip_markdown") mode="application/xml"}} + {{f.codemirror-field "csv" label=(t "admin.api.settings.error_templates_csv") tooltip=(t "admin.api.settings.error_templates_csv_tooltip_markdown") mode="text/plain"}} {{/fields-for}} {{#fields-for model=model.settings.errorDataYamlStrings style="horizontal-wide-labels" as |f|}}

{{t "mongoid.attributes.api/settings.error_data"}}

- {{f.ace-field "common" label=(t "admin.api.settings.error_data_common") tooltip=(t "admin.api.settings.error_data_common_tooltip_markdown") mode="yaml"}} - {{f.ace-field "api_key_missing" label=(t "admin.api.settings.error_data_api_key_missing") tooltip=(t "admin.api.settings.error_data_api_key_missing_tooltip_markdown") mode="yaml"}} - {{f.ace-field "api_key_invalid" label=(t "admin.api.settings.error_data_api_key_invalid") tooltip=(t "admin.api.settings.error_data_api_key_invalid_tooltip_markdown") mode="yaml"}} - {{f.ace-field "api_key_disabled" label=(t "admin.api.settings.error_data_api_key_disabled") tooltip=(t "admin.api.settings.error_data_api_key_disabled_tooltip_markdown") mode="yaml"}} - {{f.ace-field "api_key_unauthorized" label=(t "admin.api.settings.error_data_api_key_unauthorized") tooltip=(t "admin.api.settings.error_data_api_key_unauthorized_tooltip_markdown") mode="yaml"}} - {{f.ace-field "over_rate_limit" label=(t "admin.api.settings.error_data_over_rate_limit") tooltip=(t "admin.api.settings.error_data_over_rate_limit_tooltip_markdown") mode="yaml"}} - {{f.ace-field "https_required" label=(t "admin.api.settings.error_data_https_required") tooltip=(t "admin.api.settings.error_data_https_required_tooltip_markdown") mode="yaml"}} + {{f.codemirror-field "common" label=(t "admin.api.settings.error_data_common") tooltip=(t "admin.api.settings.error_data_common_tooltip_markdown") mode="text/x-yaml"}} + {{f.codemirror-field "api_key_missing" label=(t "admin.api.settings.error_data_api_key_missing") tooltip=(t "admin.api.settings.error_data_api_key_missing_tooltip_markdown") mode="text/x-yaml"}} + {{f.codemirror-field "api_key_invalid" label=(t "admin.api.settings.error_data_api_key_invalid") tooltip=(t "admin.api.settings.error_data_api_key_invalid_tooltip_markdown") mode="text/x-yaml"}} + {{f.codemirror-field "api_key_disabled" label=(t "admin.api.settings.error_data_api_key_disabled") tooltip=(t "admin.api.settings.error_data_api_key_disabled_tooltip_markdown") mode="text/x-yaml"}} + {{f.codemirror-field "api_key_unauthorized" label=(t "admin.api.settings.error_data_api_key_unauthorized") tooltip=(t "admin.api.settings.error_data_api_key_unauthorized_tooltip_markdown") mode="text/x-yaml"}} + {{f.codemirror-field "over_rate_limit" label=(t "admin.api.settings.error_data_over_rate_limit") tooltip=(t "admin.api.settings.error_data_over_rate_limit_tooltip_markdown") mode="text/x-yaml"}} + {{f.codemirror-field "https_required" label=(t "admin.api.settings.error_data_https_required") tooltip=(t "admin.api.settings.error_data_https_required_tooltip_markdown") mode="text/x-yaml"}} {{/fields-for}}
- +
{{#if model.id}} @@ -107,7 +107,7 @@
{{#if model.id}}
- Delete API + {{fa-icon "times"}}Delete API
{{/if}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/rewrite-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/rewrite-form.hbs index cadf430b4..4a9924e2b 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/rewrite-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/rewrite-form.hbs @@ -1,4 +1,4 @@ -{{#bs-modal open=openModal onSubmit=(action "submit") onHidden=(action "closed") backdropClose=false size="lg" as |modal|}} +{{#bs-modal open=openModal onSubmit=(action "submit") onHidden=(action "closed") backdropClose=false size="lg" fade=false as |modal|}} {{modal.header title=modalTitle}} {{#modal.body}}
@@ -9,18 +9,22 @@
-
+
{{f.select-field "httpMethod" label="HTTP Method" options=httpMethodOptions}}
-
+
{{f.text-field "frontendMatcher" label="Frontend Matcher" placeholder="/example/"}}
-
- rewrite to +
+
+
+ {{fa-icon "arrow-down" size="2x"}} rewrite to +
+
-
+
{{f.text-field "backendReplacement" label="Backend Replacement" placeholder="/example/"}}
diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/rewrite-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/rewrite-table.hbs index f2b73c30c..85ccde555 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/rewrite-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/rewrite-table.hbs @@ -1,10 +1,10 @@ {{apis/rewrite-form model=rewriteModel collection=model.rewrites openModal=openModal}} - +
- - + + @@ -14,16 +14,16 @@ {{#if model.rewrites}} {{#each model.rewrites as |rewrite|}} - + - + {{/each}} {{else}} @@ -33,11 +33,11 @@
Matching TypeHTTP MethodMatching TypeHTTP Method From To
{{rewrite.matcherType}} {{rewrite.httpMethod}} {{rewrite.frontendMatcher}} {{rewrite.backendReplacement}} - {{t "admin.edit"}} - {{t "admin.remove"}} + {{fa-icon "pencil-alt"}}{{t "admin.edit"}} + {{fa-icon "times"}}{{t "admin.remove"}} {{fa-icon "bars"}}
- +
{{#if isReorderable}} - + {{/if}}
diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/server-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/server-form.hbs index 3b4923980..2b1096f85 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/server-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/server-form.hbs @@ -1,4 +1,4 @@ -{{#bs-modal open=openModal onShow=(action "open") onSubmit=(action "submit") onHidden=(action "closed") backdropClose=false size="lg" as |modal|}} +{{#bs-modal open=openModal onShow=(action "open") onSubmit=(action "submit") onHidden=(action "closed") backdropClose=false size="lg" fade=false as |modal|}} {{modal.header title=modalTitle}} {{#modal.body}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/server-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/server-table.hbs index 4bb462135..77d4ed667 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/server-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/server-table.hbs @@ -1,6 +1,6 @@ {{apis/server-form model=serverModel collection=model.servers apiBackendProtocol=model.backendProtocol apiBackendHost=model.backendHost openModal=openModal}} - +
@@ -13,8 +13,8 @@ {{/each}} @@ -23,4 +23,4 @@ {{/if}}
{{t "admin.api.servers.server"}}
{{model.backendProtocol}}://{{server.hostWithPort}} - {{t "admin.edit"}} - {{t "admin.remove"}} + {{fa-icon "pencil-alt"}}{{t "admin.edit"}} + {{fa-icon "times"}}{{t "admin.remove"}}
- + diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/settings/rate-limit-fields.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/settings/rate-limit-fields.hbs index 30f66e7e0..02ae32fc9 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/settings/rate-limit-fields.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/settings/rate-limit-fields.hbs @@ -7,7 +7,7 @@ {{#if model.isRateLimitModeCustom}} {{#form-fields/field-wrapper style=(if style style "horizontal")}} - +
@@ -16,12 +16,14 @@ @@ -29,27 +31,32 @@ {{#if model.rateLimits}} {{#each model.rateLimits as |rateLimit|}} - + {{/each}} @@ -58,7 +65,7 @@ {{/if}}
DurationLimit Primary - +
- {{input type="text" value=rateLimit.durationInUnits class="form-control input-xs rate-limit-duration-in-units"}} + {{input type="text" value=rateLimit.durationInUnits class="form-control form-control-sm rate-limit-duration-in-units"}} - {{select-menu value=rateLimit.durationUnits action=(action (mut rateLimit.durationUnits)) options=rateLimitDurationUnitOptions inputClass="form-control input-xs rate-limit-duration-units"}} + {{select-menu value=rateLimit.durationUnits action=(action (mut rateLimit.durationUnits)) options=rateLimitDurationUnitOptions inputClass="custom-select custom-select-sm rate-limit-duration-units"}} - {{select-menu value=rateLimit.limitBy action=(action (mut rateLimit.limitBy)) options=rateLimitLimitByOptions inputClass="form-control input-xs rate-limit-limit-by"}} + {{select-menu value=rateLimit.limitBy action=(action (mut rateLimit.limitBy)) options=rateLimitLimitByOptions inputClass="custom-select custom-select-sm rate-limit-limit-by"}} -
- {{input type="text" value=rateLimit.limit class="form-control input-xs rate-limit-limit"}} - requests +
+ {{input type="text" value=rateLimit.limit class="form-control rate-limit-limit"}} +
+ requests +
- {{one-way-radio name=uniqueSettingsId value=rateLimit.responseHeaders option=true class="rate-limit-response-headers" update=(action "primaryRateLimitChange" rateLimit)}} +
+ {{one-way-radio name=uniqueSettingsId id=(concat uniqueSettingsId "_" rateLimit.uniqueId) value=rateLimit.responseHeaders option=true class="rate-limit-response-headers custom-control-input" update=(action "primaryRateLimitChange" rateLimit)}} + +
- Remove + {{fa-icon "times"}}Remove
- + {{/form-fields/field-wrapper}} {{/if}} {{/fields-for}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/sub-settings-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/sub-settings-form.hbs index 849644a51..583d09ab7 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/sub-settings-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/sub-settings-form.hbs @@ -1,17 +1,18 @@ -{{#bs-modal open=openModal onSubmit=(action "submit") onHidden=(action "closed") backdropClose=false size="lg" as |modal|}} +{{#bs-modal open=openModal onSubmit=(action "submit") onHidden=(action "closed") backdropClose=false size="xl" fade=false as |modal|}} {{modal.header title=modalTitle}} {{#modal.body}} {{#fields-for model=bufferedModel style="vertical" as |f|}}
-
+
{{f.select-field "httpMethod" label="HTTP Method" options=httpMethodOptions}}
-
+
{{f.text-field "regex" label="Regex" placeholder="^/example.*param1=.+"}}
{{/fields-for}} +
{{apis/settings/common-fields model=model.settings roleOptions=roleOptions isSubSettings=true}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/sub-settings-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/sub-settings-table.hbs index b242dbd62..28f96eafb 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/sub-settings-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/sub-settings-table.hbs @@ -1,9 +1,9 @@ {{apis/sub-settings-form model=subSettingsModel collection=model.subSettings roleOptions=roleOptions openModal=openModal}} - +
- + @@ -12,14 +12,14 @@ {{#if model.subSettings}} {{#each model.subSettings as |subSettings|}} - + - + {{/each}} {{else}} @@ -29,11 +29,11 @@
HTTP MethodHTTP Method URL Matcher
{{subSettings.httpMethod}} {{subSettings.regex}} - Edit - Remove + {{fa-icon "pencil-alt"}}Edit + {{fa-icon "times"}}Remove {{fa-icon "bars"}}
- +
{{#if isReorderable}} - + {{/if}}
diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/url-match-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/url-match-form.hbs index cd14d2e1c..b75809b26 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/url-match-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/url-match-form.hbs @@ -1,4 +1,4 @@ -{{#bs-modal open=openModal onSubmit=(action "submit") onHidden=(action "closed") backdropClose=false size="lg" as |modal|}} +{{#bs-modal open=openModal onSubmit=(action "submit") onHidden=(action "closed") backdropClose=false size="lg" fade=false as |modal|}} {{modal.header title=modalTitle}} {{#modal.body}}
@@ -8,7 +8,7 @@ {{f.text-field "frontendPrefix" label="Frontend Prefix" placeholder="/example/"}}
- + {{fa-icon "arrow-right" size="2x"}}
rewrite to
diff --git a/src/api-umbrella/admin-ui/app/templates/components/apis/url-match-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/apis/url-match-table.hbs index e6ca07151..b2890cf26 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/apis/url-match-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/apis/url-match-table.hbs @@ -1,7 +1,7 @@ {{apis/url-match-form model=urlMatchModel collection=model.urlMatches apiExampleIncomingUrlRoot=model.exampleIncomingUrlRoot apiExampleOutgoingUrlRoot=model.exampleOutgoingUrlRoot openModal=openModal}}
- +
@@ -13,28 +13,28 @@ {{#if model.urlMatches}} {{#each model.urlMatches as |urlMatch|}} - + - + {{/each}} {{else}} - + {{/if}}
Frontend Prefix
{{urlMatch.frontendPrefix}} {{urlMatch.backendPrefixWithDefault}} - {{t "admin.edit"}} - {{t "admin.remove"}} + {{fa-icon "pencil-alt"}}{{t "admin.edit"}} + {{fa-icon "times"}}{{t "admin.remove"}} {{fa-icon "bars"}}
{{t "admin.api.url_matches.empty_list" add=(t "admin.api.url_matches.add")}}
{{t "admin.api.url_matches.empty_list" add=(t "admin.api.url_matches.add")}}
-
- +
+
-
+
{{#if isReorderable}} - + {{/if}}
diff --git a/src/api-umbrella/admin-ui/app/templates/components/config/publish-form-records.hbs b/src/api-umbrella/admin-ui/app/templates/components/config/publish-form-records.hbs index 4686c4389..070b9181c 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/config/publish-form-records.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/config/publish-form-records.hbs @@ -1,5 +1,5 @@ {{#if records}} - +
@@ -10,18 +10,21 @@ {{#each records as |record|}} - + diff --git a/src/api-umbrella/admin-ui/app/templates/components/config/publish-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/config/publish-form.hbs index 52c4d8ca7..9b5beb1a5 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/config/publish-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/config/publish-form.hbs @@ -1,5 +1,5 @@ {{#if hasChanges}} - + {{#if model.config.apis.new}}
{{model.config.apis.new.length}} New API Backends @@ -43,11 +43,11 @@ {{/if}}
- {{t "admin.check_all"}} + {{t "admin.check_all"}}
- +
{{else}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/error-messages.hbs b/src/api-umbrella/admin-ui/app/templates/components/error-messages.hbs index 170b82c1a..c8aa5a806 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/error-messages.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/error-messages.hbs @@ -4,7 +4,7 @@

Oops! Please check the errors below:

    {{#each messages as |message|}} -
  • {{{message}}}
    • +
  • {{! template-lint-disable no-bare-strings no-triple-curlies }}{{{message}}}
    • {{/each}}
diff --git a/src/api-umbrella/admin-ui/app/templates/components/fields-for.hbs b/src/api-umbrella/admin-ui/app/templates/components/fields-for.hbs index 90952e761..b36e90b8a 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/fields-for.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/fields-for.hbs @@ -1,7 +1,7 @@ {{yield (hash - ace-field=(component "form-fields/ace-field" model=model style=style) checkbox-field=(component "form-fields/checkbox-field" model=model style=style) checkboxes-field=(component "form-fields/checkboxes-field" model=model style=style) + codemirror-field=(component "form-fields/codemirror-field" model=model style=style) select-field=(component "form-fields/select-field" model=model style=style) selectize-field=(component "form-fields/selectize-field" model=model style=style) text-field=(component "form-fields/text-field" model=model style=style) diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/checkbox-field.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/checkbox-field.hbs index 4a952de4e..5165ae242 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/checkbox-field.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/checkbox-field.hbs @@ -1,5 +1,6 @@ {{#form-fields/field-wrapper model=model style=style fieldName=fieldName inputId=inputId}} -
- +
+ {{one-way-checkbox checked=(get model fieldName) update=(action (mut (get model fieldName))) id=inputId class="custom-control-input"}} +
{{/form-fields/field-wrapper}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/checkboxes-field.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/checkboxes-field.hbs index 6e59b061c..04e822bde 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/checkboxes-field.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/checkboxes-field.hbs @@ -1,7 +1,8 @@ {{#form-fields/field-wrapper model=model style=style fieldName=fieldName inputId=inputId label=label tooltip=tooltip hint=hint}} {{#multiselect-checkboxes options=options selection=(get model fieldName) tagName="div" valueProperty="id" as |option isSelected|}} -
- +
+ {{input type="checkbox" checked=isSelected id=(concat inputId "-" option.id) class="custom-control-input"}} +
{{/multiselect-checkboxes}} {{/form-fields/field-wrapper}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/codemirror-field.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/codemirror-field.hbs new file mode 100644 index 000000000..c1e39685d --- /dev/null +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/codemirror-field.hbs @@ -0,0 +1,3 @@ +{{#form-fields/field-wrapper model=model style=style fieldName=fieldName inputId=inputId labelForId=codemirrorInputFieldId label=label tooltip=tooltip hint=hint}} + {{one-way-textarea value=(get model fieldName) update=(action (mut (get model fieldName))) class="form-control" data-codemirror-mode=mode}} +{{/form-fields/field-wrapper}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/error-messages.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/error-messages.hbs index c8b01a724..30da4981f 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/error-messages.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/error-messages.hbs @@ -1,5 +1,5 @@ {{#if hasErrors}} -
+
{{#each errorMessages as |message|}} {{message}}
{{/each}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/field-wrapper.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/field-wrapper.hbs index 29b70393b..4194943af 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/field-wrapper.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/field-wrapper.hbs @@ -1,8 +1,10 @@ {{#if (eq style "horizontal")}}
-
-
- +
+
+ {{#if label}} + + {{/if}} {{help-tooltip tooltip=tooltip}}
@@ -14,9 +16,11 @@
{{else if (eq style "horizontal-wide-labels")}}
-
-
- +
+
+ {{#if label}} + + {{/if}} {{help-tooltip tooltip=tooltip}}
@@ -28,7 +32,9 @@
{{else}}
- + {{#if label}} + + {{/if}} {{help-tooltip tooltip=tooltip}} {{yield}} {{form-fields/hint hint=hint}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/hint.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/hint.hbs index e8a886953..cce14cca3 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/hint.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/hint.hbs @@ -1,3 +1,3 @@ {{#if hint}} -

{{hint}}

+ {{hint}} {{/if}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/password-field.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/password-field.hbs index ced04717f..864898638 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/password-field.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/password-field.hbs @@ -1,3 +1,3 @@ {{#form-fields/field-wrapper model=model style=style fieldName=fieldName inputId=inputId label=label tooltip=tooltip hint=hint}} - + {{/form-fields/field-wrapper}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/select-field.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/select-field.hbs index a81845eba..f8f28e94e 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/select-field.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/select-field.hbs @@ -1,3 +1,3 @@ {{#form-fields/field-wrapper model=model style=style fieldName=fieldName inputId=inputId label=label tooltip=tooltip hint=hint}} - {{select-menu value=(get model fieldName) action=(action (mut (get model fieldName))) options=options inputId=inputId inputClass="form-control"}} + {{select-menu value=(get model fieldName) action=(action (mut (get model fieldName))) options=options inputId=inputId inputClass="custom-select"}} {{/form-fields/field-wrapper}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/selectize-field.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/selectize-field.hbs index b2d020fcc..56ebd3183 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/selectize-field.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/selectize-field.hbs @@ -1,3 +1,3 @@ {{#form-fields/field-wrapper model=model style=style fieldName=fieldName inputId=inputId labelForId=selectizeTextInputId label=label tooltip=tooltip hint=hint}} - + {{/form-fields/field-wrapper}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/static-field.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/static-field.hbs index 17c5d01b7..c2464b7e9 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/static-field.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/static-field.hbs @@ -1,5 +1,5 @@ {{#form-fields/field-wrapper model=model style=style fieldName=fieldName inputId=inputId label=label tooltip=tooltip hint=hint}} -
+
{{#if hasBlock}} {{yield}} {{else}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/text-field.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/text-field.hbs index f067c910c..c35d21499 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/text-field.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/text-field.hbs @@ -1,3 +1,3 @@ {{#form-fields/field-wrapper model=model style=style fieldName=fieldName inputId=inputId label=label tooltip=tooltip hint=hint}} - + {{/form-fields/field-wrapper}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/form-fields/textarea-field.hbs b/src/api-umbrella/admin-ui/app/templates/components/form-fields/textarea-field.hbs index 2969dc339..da8e2fe9e 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/form-fields/textarea-field.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/form-fields/textarea-field.hbs @@ -1,3 +1,3 @@ {{#form-fields/field-wrapper model=model style=style fieldName=fieldName inputId=inputId label=label tooltip=tooltip hint=hint}} - + {{/form-fields/field-wrapper}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/help-tooltip.hbs b/src/api-umbrella/admin-ui/app/templates/components/help-tooltip.hbs index 0e7e05890..7987f4e31 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/help-tooltip.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/help-tooltip.hbs @@ -1,3 +1,3 @@ {{#if tooltip}} - Help + {{/if}} diff --git a/src/api-umbrella/admin-ui/app/templates/components/stats/drilldown/results-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/stats/drilldown/results-table.hbs index acd08a54e..804b0605f 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/stats/drilldown/results-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/stats/drilldown/results-table.hbs @@ -1,6 +1,6 @@
-
Publish?
- - - - - + + + + +
+ + +
{{record.name}}
View Config Differences
+
- Download CSV + Download CSV
diff --git a/src/api-umbrella/admin-ui/app/templates/components/stats/logs/results-facet-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/stats/logs/results-facet-table.hbs index 2edda9c4a..f250c61c2 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/stats/logs/results-facet-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/stats/logs/results-facet-table.hbs @@ -1,5 +1,5 @@ -
{{label}}
- +
{{label}}
+ @@ -10,7 +10,7 @@ {{#each facets as |facet|}} - + diff --git a/src/api-umbrella/admin-ui/app/templates/components/stats/logs/results-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/stats/logs/results-table.hbs index a065383c0..b936e6aa5 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/stats/logs/results-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/stats/logs/results-table.hbs @@ -1,6 +1,6 @@
-
+
- Download CSV + Download CSV
diff --git a/src/api-umbrella/admin-ui/app/templates/components/stats/map/results-table.hbs b/src/api-umbrella/admin-ui/app/templates/components/stats/map/results-table.hbs index acd08a54e..804b0605f 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/stats/map/results-table.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/stats/map/results-table.hbs @@ -1,6 +1,6 @@
-
+
- Download CSV + Download CSV
diff --git a/src/api-umbrella/admin-ui/app/templates/components/stats/query-form.hbs b/src/api-umbrella/admin-ui/app/templates/components/stats/query-form.hbs index 0faf06903..5426cf6de 100644 --- a/src/api-umbrella/admin-ui/app/templates/components/stats/query-form.hbs +++ b/src/api-umbrella/admin-ui/app/templates/components/stats/query-form.hbs @@ -1,68 +1,69 @@
- {{t "admin.stats.filter_results"}} +
+ {{fa-icon "caret-down" fixedWidth=true}}{{t "admin.stats.filter_results"}} +
- {{#if session.data.authenticated.enable_beta_analytics}} - - {{/if}} - {{#if enableInterval}} - {{#bs-button-group value=interval onChange=(action (mut interval)) type="radio" size="xs" id="interval_buttons" as |bg|}} - {{#bg.button value="minute"}}{{t "admin.stats.minute"}}{{/bg.button}} - {{#bg.button value="hour"}}{{t "admin.stats.hour"}}{{/bg.button}} - {{#bg.button value="day"}}{{t "admin.stats.day"}}{{/bg.button}} - {{#bg.button value="week"}}{{t "admin.stats.week"}}{{/bg.button}} - {{#bg.button value="month"}}{{t "admin.stats.month"}}{{/bg.button}} + {{#bs-button-group value=interval onChange=(action (mut interval)) type="radio" size="sm" id="interval_buttons" as |bg|}} + {{#bg.button type="light" value="minute"}}{{t "admin.stats.minute"}}{{/bg.button}} + {{#bg.button type="light" value="hour"}}{{t "admin.stats.hour"}}{{/bg.button}} + {{#bg.button type="light" value="day"}}{{t "admin.stats.day"}}{{/bg.button}} + {{#bg.button type="light" value="week"}}{{t "admin.stats.week"}}{{/bg.button}} + {{#bg.button type="light" value="month"}}{{t "admin.stats.month"}}{{/bg.button}} {{/bs-button-group}} {{/if}}
-