Integrate Vector for log management, update Docker actions, and add d…

…atabase migration steps.
lostlink · Oct 1, 2024 · 5a1c477 · 5a1c477
1 parent 89dfe99
commit 5a1c477
Show file tree

Hide file tree

Showing 9 changed files with 156 additions and 14 deletions.
diff --git a/.github/workflows/octane-pdns.yaml b/.github/workflows/octane-pdns.yaml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
 
@@ -55,7 +55,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
 
@@ -66,19 +66,19 @@ jobs:
           expand: "true"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: "{{defaultContext}}:${{ env.APP_NAME }}"
           platforms: ${{ matrix.build.platforms }}
@@ -92,6 +92,7 @@ jobs:
             "WWWGROUP=${{ env.WWWGROUP }}"
             "ENABLE_HORIZON=${{ env.ENABLE_HORIZON }}"
             "ENABLE_SCHEDULER=${{ env.ENABLE_SCHEDULER }}"
+            "ENABLE_VECTOR=${{ env.ENABLE_VECTOR }}"
             "ENABLE_PDNS=${{ env.ENABLE_PDNS }}"
             "INCLUDE_NPM=${{ env.INCLUDE_NPM }}"
             "INSTALL_OPCACHE=${{ env.INSTALL_OPCACHE }}"

diff --git a/.github/workflows/octane.yaml b/.github/workflows/octane.yaml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
 
@@ -68,7 +68,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           lfs: true
 
@@ -79,19 +79,19 @@ jobs:
           expand: "true"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v6
         with:
           context: "{{defaultContext}}:${{ env.APP_NAME }}"
           platforms: ${{ matrix.build.platforms }}
@@ -105,6 +105,7 @@ jobs:
             "WWWGROUP=${{ env.WWWGROUP }}"
             "ENABLE_HORIZON=${{ env.ENABLE_HORIZON }}"
             "ENABLE_SCHEDULER=${{ env.ENABLE_SCHEDULER }}"
+            "ENABLE_VECTOR=${{ env.ENABLE_VECTOR }}"
             "INCLUDE_NPM=${{ env.INCLUDE_NPM }}"
             "INSTALL_OPCACHE=${{ env.INSTALL_OPCACHE }}"
             "INSTALL_PHPREDIS=${{ env.INSTALL_PHPREDIS }}"

diff --git a/octane-pdns/Dockerfile b/octane-pdns/Dockerfile
@@ -23,6 +23,9 @@ RUN composer require laravel/octane && \
 
 RUN sed -i 's/LOG_CHANNEL=stack/LOG_CHANNEL=stderr/g' .env
 
+RUN touch database/database.sqlite && \
+    php artisan migrate --force
+
 ###########################################
 # Octane
 ###########################################
@@ -35,6 +38,7 @@ ARG WWWGROUP=1000
 ARG TIMEZONE=UTC
 ARG ENABLE_HORIZON=false
 ARG ENABLE_SCHEDULER=false
+ARG ENABLE_VECTOR=false
 ARG ENABLE_PDNS=true
 ARG TARGETARCH
 ARG NODE_VERSION
@@ -65,6 +69,7 @@ ENV DEBIAN_FRONTEND=noninteractive \
     TERM=xterm-color \
     ENABLE_HORIZON=${ENABLE_HORIZON} \
     ENABLE_SCHEDULER=${ENABLE_SCHEDULER} \
+    ENABLE_VECTOR=${ENABLE_VECTOR} \
     ENABLE_PDNS=${ENABLE_PDNS} \
     COMPOSER_ALLOW_SUPERUSER=1
 
@@ -291,6 +296,20 @@ RUN wget -q "https://github.com/aptible/supercronic/releases/download/v0.2.1/sup
     echo "*/1 * * * * php ${ROOT}/artisan schedule:run --verbose --no-interaction" > /etc/supercronic/laravel;
 ###########################################
 
+###########################################
+# Install Vector using .deb package
+###########################################
+RUN curl -LO https://packages.timber.io/vector/0.41.1/vector_0.41.1-1_${TARGETARCH}.deb \
+    && dpkg -i vector_0.41.1-1_${TARGETARCH}.deb \
+    && rm vector_0.41.1-1_${TARGETARCH}.deb
+
+# Create the Vector configuration directory
+RUN mkdir -p /etc/vector
+
+# Copy the Vector configuration file
+COPY ./etc/vector/vector.toml /etc/vector/vector.toml
+###########################################
+
 RUN groupadd --force -g $WWWGROUP octane && \
     useradd -ms /bin/bash --no-log-init --no-user-group -g $WWWGROUP -u $WWWUSER octane
 
@@ -317,6 +336,8 @@ RUN apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     rm /var/log/lastlog /var/log/faillog
 
-EXPOSE 9000 53 53/udp
+RUN chown -R octane:octane /var/www/html
+
+EXPOSE 9000 9001 9002 53 53/udp
 
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/octane-pdns/etc/supervisor/conf.d/vector.conf b/octane-pdns/etc/supervisor/conf.d/vector.conf
@@ -0,0 +1,10 @@
+[program:vector]
+process_name=%(program_name)s
+command=/usr/bin/vector --config /etc/vector/vector.toml
+user=octane
+autostart=%(ENV_ENABLE_VECTOR)s
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
diff --git a/octane-pdns/etc/vector/vector.toml b/octane-pdns/etc/vector/vector.toml
@@ -0,0 +1,40 @@
+# TCP source for incoming JSON data
+[sources.socket_tcp]
+  type = "socket"
+  address = "0.0.0.0:9001"
+  mode = "tcp"
+  max_length = 1024000
+
+# UDP source for incoming JSON data
+[sources.socket_udp]
+  type = "socket"
+  address = "0.0.0.0:9002"
+  mode = "udp"
+  max_length = 1024000
+
+# Transform: Parse incoming JSON and isolate the message field
+[transforms.parse_json]
+  type = "remap"
+  inputs = ["socket_tcp", "socket_udp"]
+  source = '''
+  .kinesis_stream_name = .kinesis_stream_name
+  .aws_region = .aws_region
+  . = parse_json!(.message)
+  '''
+
+# Kinesis sink with dynamically set stream name and region
+[sinks.kinesis_dynamic]
+  type = "aws_kinesis_streams"
+  inputs = ["parse_json"]
+  stream_name = "{{kinesis_stream_name}}"
+  region = "{{aws_region}}"
+  encoding.codec = "json"
+
+  # Optional: Configure batching to optimize network usage
+  batch.max_bytes = 1024000
+  batch.timeout_secs = 1
+
+  # Retry settings
+  request.in_flight_limit = 5
+  request.retry_attempts = 10
+  request.timeout_secs = 30
diff --git a/octane/Dockerfile b/octane/Dockerfile
@@ -22,6 +22,9 @@ RUN composer require laravel/octane && \
 
 RUN sed -i 's/LOG_CHANNEL=stack/LOG_CHANNEL=stderr/g' .env
 
+RUN touch database/database.sqlite && \
+    php artisan migrate --force
+
 ###########################################
 # Octane
 ###########################################
@@ -34,6 +37,7 @@ ARG WWWGROUP=1000
 ARG TIMEZONE=UTC
 ARG ENABLE_HORIZON=false
 ARG ENABLE_SCHEDULER=false
+ARG ENABLE_VECTOR=false
 ARG TARGETARCH
 ARG NODE_VERSION
 ARG INCLUDE_NPM=true
@@ -61,6 +65,7 @@ ENV DEBIAN_FRONTEND=noninteractive \
     TERM=xterm-color \
     ENABLE_HORIZON=${ENABLE_HORIZON} \
     ENABLE_SCHEDULER=${ENABLE_SCHEDULER} \
+    ENABLE_VECTOR=${ENABLE_VECTOR} \
     COMPOSER_ALLOW_SUPERUSER=1
 
 ENV ROOT=/var/www/html
@@ -256,6 +261,20 @@ RUN wget -q "https://github.com/aptible/supercronic/releases/download/v0.2.1/sup
     echo "*/1 * * * * php ${ROOT}/artisan schedule:run --verbose --no-interaction" > /etc/supercronic/laravel;
 ###########################################
 
+###########################################
+# Install Vector using .deb package
+###########################################
+RUN curl -LO https://packages.timber.io/vector/0.41.1/vector_0.41.1-1_${TARGETARCH}.deb \
+    && dpkg -i vector_0.41.1-1_${TARGETARCH}.deb \
+    && rm vector_0.41.1-1_${TARGETARCH}.deb
+
+# Create the Vector configuration directory
+RUN mkdir -p /etc/vector
+
+# Copy the Vector configuration file
+COPY ./etc/vector/vector.toml /etc/vector/vector.toml
+###########################################
+
 RUN groupadd --force -g $WWWGROUP octane && \
     useradd -ms /bin/bash --no-log-init --no-user-group -g $WWWGROUP -u $WWWUSER octane
 
@@ -279,6 +298,8 @@ RUN apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
     rm /var/log/lastlog /var/log/faillog
 
-EXPOSE 9000
+RUN chown -R octane:octane /var/www/html
+
+EXPOSE 9000 9001 9002
 
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/octane/docker-compose.yml b/octane/docker-compose.yml
@@ -1,5 +1,3 @@
-version: '3.7'
-
 services:
   octane:
     build:

diff --git a/octane/etc/supervisor/conf.d/vector.conf b/octane/etc/supervisor/conf.d/vector.conf
@@ -0,0 +1,10 @@
+[program:vector]
+process_name=%(program_name)s
+command=/usr/bin/vector --config /etc/vector/vector.toml
+user=octane
+autostart=%(ENV_ENABLE_VECTOR)s
+autorestart=true
+stdout_logfile=/dev/stdout
+stdout_logfile_maxbytes=0
+stderr_logfile=/dev/stderr
+stderr_logfile_maxbytes=0
diff --git a/octane/etc/vector/vector.toml b/octane/etc/vector/vector.toml
@@ -0,0 +1,40 @@
+# TCP source for incoming JSON data
+[sources.socket_tcp]
+  type = "socket"
+  address = "0.0.0.0:9001"
+  mode = "tcp"
+  max_length = 1024000
+
+# UDP source for incoming JSON data
+[sources.socket_udp]
+  type = "socket"
+  address = "0.0.0.0:9002"
+  mode = "udp"
+  max_length = 1024000
+
+# Transform: Parse incoming JSON and isolate the message field
+[transforms.parse_json]
+  type = "remap"
+  inputs = ["socket_tcp", "socket_udp"]
+  source = '''
+  .kinesis_stream_name = .kinesis_stream_name
+  .aws_region = .aws_region
+  . = parse_json!(.message)
+  '''
+
+# Kinesis sink with dynamically set stream name and region
+[sinks.kinesis_dynamic]
+  type = "aws_kinesis_streams"
+  inputs = ["parse_json"]
+  stream_name = "{{kinesis_stream_name}}"
+  region = "{{aws_region}}"
+  encoding.codec = "json"
+
+  # Optional: Configure batching to optimize network usage
+  batch.max_bytes = 1024000
+  batch.timeout_secs = 1
+
+  # Retry settings
+  request.in_flight_limit = 5
+  request.retry_attempts = 10
+  request.timeout_secs = 30