Review the generation of the datadog.conf to make it more flexible and maintainable

Dockerfile

@@ -3,7 +3,13 @@ FROM debian:jessie
 MAINTAINER Datadog <package@datadoghq.com>
 
 ENV DOCKER_DD_AGENT=yes \
-    AGENT_VERSION=1:5.16.0-1
+    AGENT_VERSION=1:5.16.0-1 \
+    DD_ETC_ROOT=/etc/dd-agent \
+    PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:${PATH}" \
+    PYTHONPATH=/opt/datadog-agent/agent \
+    DD_CONF_LOG_TO_SYSLOG=no \
+    NON_LOCAL_TRAFFIC=yes \
+    DD_SUPERVISOR_DELETE_USER=yes
 
 # Install the Agent
 RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/datadog.list \
@@ -15,23 +21,18 @@ RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/d
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Configure the Agent
-# 1. Listen to statsd (8125) and traces (8126) from other containers
-# 2. Turn syslog off
-# 3. Remove dd-agent user from supervisor configuration
-# 4. Remove dd-agent user from init.d configuration
-# 5. Fix permission on /etc/init.d/datadog-agent
-RUN mv /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf \
- && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" /etc/dd-agent/datadog.conf \
- && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" /etc/dd-agent/datadog.conf \
- && sed -i "/user=dd-agent/d" /etc/dd-agent/supervisor.conf \
+# 1. Remove dd-agent user from init.d configuration
+# 2. Fix permission on /etc/init.d/datadog-agent
+RUN mv ${DD_ETC_ROOT}/datadog.conf.example ${DD_ETC_ROOT}/datadog.conf \
  && sed -i 's/AGENTUSER="dd-agent"/AGENTUSER="root"/g' /etc/init.d/datadog-agent \
- && rm /etc/dd-agent/conf.d/network.yaml.default \
- || chmod +x /etc/init.d/datadog-agent
+ && rm -f ${DD_ETC_ROOT}/conf.d/network.yaml.default \
+ && chmod +x /etc/init.d/datadog-agent
 
 # Add Docker check
-COPY conf.d/docker_daemon.yaml /etc/dd-agent/conf.d/docker_daemon.yaml
-
+COPY conf.d/docker_daemon.yaml ${DD_ETC_ROOT}/conf.d/docker_daemon.yaml
+# Add install and config files
 COPY entrypoint.sh /entrypoint.sh
+COPY config_builder.py /config_builder.py
 
 # Extra conf.d and checks.d
 VOLUME ["/conf.d", "/checks.d"]

alpine/Dockerfile → Dockerfile-alpine

@@ -7,50 +7,49 @@ ENV DD_HOME=/opt/datadog-agent \
     DD_START_AGENT=0 \
     DOCKER_DD_AGENT=yes \
     PYCURL_SSL_LIBRARY=openssl \
-    AGENT_VERSION=5.16.0
-
-# Add Docker check
-COPY conf.d/docker_daemon.yaml "$DD_HOME/agent/conf.d/docker_daemon.yaml"
-
-# Add install and config files
-ADD https://raw.githubusercontent.com/DataDog/dd-agent/master/packaging/datadog-agent/source/setup_agent.sh /tmp/setup_agent.sh
-COPY entrypoint.sh /entrypoint.sh
-
-# Expose supervisor and DogStatsD port
-EXPOSE 9001/tcp 8125/udp
+    AGENT_VERSION=5.16.0 \
+    DD_ETC_ROOT="/opt/datadog-agent/agent" \
+    PATH="/opt/datadog-agent/venv/bin:/opt/datadog-agent/agent/bin:$PATH" \
+    PYTHONPATH="/opt/datadog-agent/agent" \
+    DD_CONF_LOG_TO_SYSLOG=no \
+    NON_LOCAL_TRAFFIC=yes \
+    DD_SUPERVISOR_DELETE_USER=yes
 
 # Install minimal dependencies
 RUN apk add -qU --no-cache coreutils curl curl-dev python-dev tar sysstat tini
 
 # Install build dependencies
+ADD https://raw.githubusercontent.com/DataDog/dd-agent/master/packaging/datadog-agent/source/setup_agent.sh /tmp/setup_agent.sh
 RUN apk add -qU --no-cache -t .build-deps gcc musl-dev pgcluster-dev linux-headers \
     # Install the agent
     && sh /tmp/setup_agent.sh \
     # Clean build dependencies
-    && apk del -q .build-deps
+    && apk del -q .build-deps \
+    && rm /tmp/setup_agent.sh
 
 # Configure the Agent
-# 1. Listen to statsd from other containers
-# 2. Turn syslog off
-# 3. Remove dd-agent user from supervisor configuration
-# 4. Remove setup script
-RUN cp "$DD_HOME/agent/datadog.conf.example" "$DD_HOME/agent/datadog.conf" \
-  && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" "$DD_HOME/agent/datadog.conf" \
-  && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" "$DD_HOME/agent/datadog.conf" \
-  && sed -i "/user=dd-agent/d" "$DD_HOME/agent/supervisor.conf" \
-  && rm "$DD_HOME/agent/conf.d/network.yaml.default" \
-  || rm /tmp/setup_agent.sh
+RUN cp ${DD_ETC_ROOT}/datadog.conf.example ${DD_ETC_ROOT}/datadog.conf \
+  && rm -f ${DD_ETC_ROOT}/conf.d/network.yaml.default
+
+# Add Docker check
+COPY conf.d/docker_daemon.yaml "${DD_ETC_ROOT}/conf.d/docker_daemon.yaml"
+# Add install and config files
+COPY entrypoint.sh /entrypoint.sh
+COPY config_builder.py /config_builder.py
+
+# Extra conf.d and checks.d
+VOLUME ["/conf.d", "/checks.d"]
+
+# Expose supervisor and DogStatsD port
+EXPOSE 9001/tcp 8125/udp
 
 # Healthcheck
 HEALTHCHECK --interval=5m --timeout=3s --retries=1 \
   CMD test $($DD_HOME/venv/bin/python $DD_HOME/venv/bin/supervisorctl \
-      -c $DD_HOME/agent/supervisor.conf status | awk '{print $2}' | egrep -v 'RUNNING|EXITED' | wc -l) \
+      -c ${DD_ETC_ROOT}/supervisor.conf status | awk '{print $2}' | egrep -v 'RUNNING|EXITED' | wc -l) \
       -eq 0 || exit 1
 
-# Extra conf.d and checks.d
-VOLUME ["/conf.d", "/checks.d"]
-
 ENTRYPOINT ["/sbin/tini", "-g", "--", "/entrypoint.sh"]
 
-WORKDIR "$DD_HOME"
-CMD source venv/bin/activate && supervisord -c agent/supervisor.conf
+WORKDIR $DD_HOME
+CMD ["supervisord", "-c", "agent/supervisor.conf"]

dogstatsd/Dockerfile → Dockerfile-dogstatsd

@@ -3,9 +3,12 @@ FROM debian:jessie
 MAINTAINER Datadog <package@datadoghq.com>
 
 ENV DOCKER_DD_AGENT=yes \
-    AGENT_VERSION=1:5.16.0-1
-
-COPY entrypoint.sh supervisor.conf /
+    AGENT_VERSION=1:5.16.0-1 \
+    DD_ETC_ROOT=/etc/dd-agent \
+    PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:${PATH}" \
+    PYTHONPATH=/opt/datadog-agent/agent \
+    DD_CONF_LOG_TO_SYSLOG=no \
+    NON_LOCAL_TRAFFIC=yes
 
 # Install the Agent
 RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/datadog.list \
@@ -15,24 +18,21 @@ RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/d
  && apt-get clean \
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# Configure the Agent
-# 1. Listen to statsd from other containers
-# 2. Turn syslog off
-# 3. Use custom supervisor.conf
-RUN mv /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf \
- && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" /etc/dd-agent/datadog.conf \
- && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" /etc/dd-agent/datadog.conf \
- && mv /supervisor.conf /etc/dd-agent/supervisor.conf
-
-# Expose supervisor, DogStatsD and trace-agent port
-EXPOSE 9001/tcp 8125/udp 8126/tcp
+# Add install and config files
+COPY entrypoint-dogstatsd.sh /entrypoint.sh
+COPY dogstatsd/supervisor.conf /etc/dd-agent/supervisor.conf
+COPY config_builder.py /config_builder.py
 
-# Set proper permissions to allow running as a non-root user
-RUN chmod g+w /etc/dd-agent/datadog.conf \
+RUN mv /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf \
+ # Set proper permissions to allow running as a non-root user
+ && chmod g+w /etc/dd-agent/datadog.conf \
  && chmod -R g+w /var/log/datadog \
  && chmod g+w /etc/dd-agent \
  && chmod g+w /opt/datadog-agent/run/
 
+# Expose supervisor, DogStatsD and trace-agent port
+EXPOSE 9001/tcp 8125/udp 8126/tcp
+
 # Healthcheck
 HEALTHCHECK --interval=5m --timeout=3s --retries=1 \
   CMD test $(/opt/datadog-agent/embedded/bin/python /opt/datadog-agent/bin/supervisorctl \

dogstatsd/alpine/Dockerfile → Dockerfile-dogstatsd-alpine

@@ -6,40 +6,39 @@ ENV DD_HOME=/opt/datadog-agent \
     DOCKER_DD_AGENT=yes \
     # prevent the agent from being started after install
     DD_START_AGENT=0 \
-    AGENT_VERSION=5.16.0
-
-# Add install and config files
-ADD https://raw.githubusercontent.com/DataDog/dd-agent/master/packaging/datadog-agent/source/setup_agent.sh /tmp/setup_agent.sh
-COPY entrypoint.sh supervisor.conf /
-
-# Expose supervisor and DogStatsD port
-EXPOSE 9001/tcp 8125/udp
+    AGENT_VERSION=5.16.0 \
+    DD_ETC_ROOT="/opt/datadog-agent/agent" \        
+    PATH="/opt/datadog-agent/venv/bin:/opt/datadog-agent/agent/bin:$PATH" \
+    PYTHONPATH=/opt/datadog-agent/agent \
+    DD_CONF_LOG_TO_SYSLOG=no \
+    NON_LOCAL_TRAFFIC=yes
 
 # Install minimal dependencies
 RUN apk add -qU --no-cache curl-dev python-dev tar sysstat ca-certificates
 
 # Install build dependencies
+ADD https://raw.githubusercontent.com/DataDog/dd-agent/master/packaging/datadog-agent/source/setup_agent.sh /tmp/setup_agent.sh
 RUN apk add -qU --no-cache -t .build-deps curl gcc musl-dev pgcluster-dev linux-headers \
     # Install the agent
     && sh /tmp/setup_agent.sh \
     # Clean build dependencies
-    && apk del -q .build-deps
-
-# Configure the Agent
-# 1. Listen to statsd from other containers
-# 2. Turn syslog off
-# 3. Use custom supervisor.conf
-# 4. Clean up the install script
-RUN mv $DD_HOME/agent/datadog.conf.example $DD_HOME/agent/datadog.conf \
- && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" $DD_HOME/agent/datadog.conf \
- && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" $DD_HOME/agent/datadog.conf \
- && mv /supervisor.conf $DD_HOME/agent/supervisor.conf \
- && rm /tmp/setup_agent.sh
+    && apk del -q .build-deps \
+    # Clean up the install script
+    && rm /tmp/setup_agent.sh
+
+# Add install and config files                      
+COPY entrypoint-dogstatsd.sh /entrypoint.sh                   
+COPY dogstatsd/supervisor-alpine.conf $DD_HOME/agent/supervisor.conf
+COPY config_builder.py /config_builder.py 
 
 # Set proper permissions to allow running as a non-root user
-RUN chmod -R g+wx $DD_HOME \
+RUN  mv $DD_HOME/agent/datadog.conf.example $DD_HOME/agent/datadog.conf \
+  && chmod -R g+wx $DD_HOME \
   && chmod g+x /entrypoint.sh
 
+# Expose supervisor and DogStatsD port
+EXPOSE 9001/tcp 8125/udp
+
 ENTRYPOINT ["/entrypoint.sh"]
 
 USER 1001

jmx/Dockerfile → Dockerfile-jmx

@@ -3,7 +3,14 @@ FROM debian:jessie
 MAINTAINER Datadog <package@datadoghq.com>
 
 ENV DOCKER_DD_AGENT=yes \
-    AGENT_VERSION=1:5.16.0-1
+    AGENT_VERSION=1:5.16.0-1 \
+    DD_ETC_ROOT=/etc/dd-agent \
+    PATH="/opt/datadog-agent/embedded/bin:/opt/datadog-agent/bin:${PATH}" \
+    PYTHONPATH=/opt/datadog-agent/agent \
+    DD_CONF_SD_JMX_ENABLE=yes \
+    DD_CONF_LOG_TO_SYSLOG=no \
+    NON_LOCAL_TRAFFIC=yes \
+    DD_SUPERVISOR_DELETE_USER=yes
 
 # Install the Agent
 RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/datadog.list \
@@ -15,25 +22,16 @@ RUN echo "deb http://apt.datadoghq.com/ stable main" > /etc/apt/sources.list.d/d
  && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Configure the Agent
-# 1. Listen to statsd from other containers
-# 2. Turn syslog off
-# 3. Enable Service Discovery with JMXFetch
-# 4. Remove dd-agent user from supervisor configuration
-# 5. Remove dd-agent user from init.d configuration
-# 6. Fix permission on /etc/init.d/datadog-agent
-RUN mv /etc/dd-agent/datadog.conf.example /etc/dd-agent/datadog.conf \
- && sed -i -e"s/^.*non_local_traffic:.*$/non_local_traffic: yes/" /etc/dd-agent/datadog.conf \
- && sed -i -e"s/^.*log_to_syslog:.*$/log_to_syslog: no/" /etc/dd-agent/datadog.conf \
- && sed -i -e"s/^.*sd_jmx_enable:.*$/sd_jmx_enable: yes/" /etc/dd-agent/datadog.conf \
- && sed -i "/user=dd-agent/d" /etc/dd-agent/supervisor.conf \
+RUN mv ${DD_ETC_ROOT}/datadog.conf.example ${DD_ETC_ROOT}/datadog.conf \
  && sed -i 's/AGENTUSER="dd-agent"/AGENTUSER="root"/g' /etc/init.d/datadog-agent \
- && rm /etc/dd-agent/conf.d/network.yaml.default \
- || chmod +x /etc/init.d/datadog-agent
+ && rm -f ${DD_ETC_ROOT}/conf.d/network.yaml.default \
+ && chmod +x /etc/init.d/datadog-agent
 
-# Add Docker check
-COPY conf.d/docker_daemon.yaml /etc/dd-agent/conf.d/docker_daemon.yaml
-
-COPY entrypoint.sh /entrypoint.sh
+# Add Docker check                                  
+COPY conf.d/docker_daemon.yaml ${DD_ETC_ROOT}/conf.d/docker_daemon.yaml                                  
+# Add install and config files                      
+COPY entrypoint.sh /entrypoint.sh                   
+COPY config_builder.py /config_builder.py
 
 # Extra conf.d and checks.d
 VOLUME ["/conf.d", "/checks.d"]
@@ -44,7 +42,7 @@ EXPOSE 8125/udp 9001/tcp
 # Healthcheck
 HEALTHCHECK --interval=5m --timeout=3s --retries=1 \
   CMD test $(/opt/datadog-agent/embedded/bin/python /opt/datadog-agent/bin/supervisorctl \
-      -c /etc/dd-agent/supervisor.conf status | awk '{print $2}' | egrep -v 'RUNNING|EXITED' | wc -l) \
+      -c ${DD_ETC_ROOT}/supervisor.conf status | awk '{print $2}' | egrep -v 'RUNNING|EXITED' | wc -l) \
       -eq 0 || exit 1
 
 ENTRYPOINT ["/entrypoint.sh"]