ci: bare-metal: poe: Create strutured logs

Use the CustomLogger class and CLI tool to create strutured logs
for poe scripts which are used by broadcom and nouveau jobs.
Renamed stage lint to code-validation and added python-test job
which runs the tests for structured and customer logger to ci.

Signed-off-by: Vignesh Raman <vignesh.raman@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25179>

.gitlab-ci.yml

@@ -85,6 +85,8 @@ variables:
   # Re-enable farm with `git mv .ci-farms{-disabled,}/$farm_name`
   # NEVER MIX FARM MAINTENANCE WITH ANY OTHER CHANGE IN THE SAME MERGE REQUEST!
   ARTIFACTS_BASE_URL: https://${CI_PROJECT_ROOT_NAMESPACE}.${CI_PAGES_DOMAIN}/-/${CI_PROJECT_NAME}/-/jobs/${CI_JOB_ID}/artifacts
+  # Python scripts for structured logger
+  PYTHONPATH: "$PYTHONPATH:$CI_PROJECT_DIR/install"
 
 default:
   before_script:
@@ -127,7 +129,7 @@ stages:
   - git-archive
   - build-x86_64
   - build-misc
-  - lint
+  - code-validation
   - amd
   - intel
   - nouveau

.gitlab-ci/bare-metal/poe-powered.sh

@@ -12,6 +12,7 @@
 # We're run from the root of the repo, make a helper var for our paths
 BM=$CI_PROJECT_DIR/install/bare-metal
 CI_COMMON=$CI_PROJECT_DIR/install/common
+CI_INSTALL=$CI_PROJECT_DIR/install
 
 # Runner config checks
 if [ -z "$BM_SERIAL" ]; then
@@ -181,8 +182,19 @@ if [ -n "$BM_BOOTCONFIG" ]; then
 fi
 
 set +e
+STRUCTURED_LOG_FILE=job_detail.json
+python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update dut_job_type "${DEVICE_TYPE}"
+python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update farm "${FARM}"
 ATTEMPTS=3
+first_attempt=True
 while [ $((ATTEMPTS--)) -gt 0 ]; do
+  python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --create-dut-job dut_name "${CI_RUNNER_DESCRIPTION}"
+  # Update subtime time to CI_JOB_STARTED_AT only for the first run
+  if [ "$first_attempt" = "True" ]; then
+    python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update-dut-time submit "${CI_JOB_STARTED_AT}"
+  else
+    python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --update-dut-time submit
+  fi
   python3 $BM/poe_run.py \
           --dev="$BM_SERIAL" \
           --powerup="$BM_POWERUP" \
@@ -192,10 +204,14 @@ while [ $((ATTEMPTS--)) -gt 0 ]; do
 
   if [ $ret -eq 2 ]; then
     echo "Did not detect boot sequence, retrying..."
+    python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --close-dut-job
+    first_attempt=False
   else
     ATTEMPTS=0
   fi
 done
+python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --close-dut-job
+python3 $CI_INSTALL/custom_logger.py ${STRUCTURED_LOG_FILE} --close
 set -e
 
 date +'%F %T'
@@ -203,6 +219,10 @@ date +'%F %T'
 # Bring artifacts back from the NFS dir to the build dir where gitlab-runner
 # will look for them.
 cp -Rp /nfs/results/. results/
+if [ -f "${STRUCTURED_LOG_FILE}" ]; then
+  cp -p ${STRUCTURED_LOG_FILE} results/
+  echo "Structured log file is available at ${ARTIFACTS_BASE_URL}/results/${STRUCTURED_LOG_FILE}"
+fi
 
 date +'%F %T'
 

.gitlab-ci/bare-metal/poe_run.py

@@ -24,23 +24,26 @@
 import argparse
 import os
 import re
-from serial_buffer import SerialBuffer
 import sys
 import threading
 
+from custom_logger import CustomLogger
+from serial_buffer import SerialBuffer
 
 class PoERun:
-    def __init__(self, args, test_timeout):
+    def __init__(self, args, test_timeout, logger):
         self.powerup = args.powerup
         self.powerdown = args.powerdown
         self.ser = SerialBuffer(
             args.dev, "results/serial-output.txt", "")
         self.test_timeout = test_timeout
+        self.logger = logger
 
     def print_error(self, message):
         RED = '\033[0;31m'
         NO_COLOR = '\033[0m'
         print(RED + message + NO_COLOR)
+        self.logger.update_status_fail(message)
 
     def logged_system(self, cmd):
         print("Running '{}'".format(cmd))
@@ -48,9 +51,11 @@ class PoERun:
 
     def run(self):
         if self.logged_system(self.powerup) != 0:
+            self.logger.update_status_fail("powerup failed")
             return 1
 
         boot_detected = False
+        self.logger.create_job_phase("boot")
         for line in self.ser.lines(timeout=5 * 60, phase="bootloader"):
             if re.search("Booting Linux", line):
                 boot_detected = True
@@ -61,8 +66,10 @@ class PoERun:
                 "Something wrong; couldn't detect the boot start up sequence")
             return 1
 
+        self.logger.create_job_phase("test")
         for line in self.ser.lines(timeout=self.test_timeout, phase="test"):
             if re.search("---. end Kernel panic", line):
+                self.logger.update_status_fail("kernel panic")
                 return 1
 
             # Binning memory problems
@@ -82,8 +89,10 @@ class PoERun:
             result = re.search("hwci: mesa: (\S*)", line)
             if result:
                 if result.group(1) == "pass":
+                    self.logger.update_dut_job("status", "pass")
                     return 0
                 else:
+                    self.logger.update_status_fail("test fail")
                     return 1
 
         self.print_error(
@@ -103,10 +112,13 @@ def main():
         '--test-timeout', type=int, help='Test phase timeout (minutes)', required=True)
     args = parser.parse_args()
 
-    poe = PoERun(args, args.test_timeout * 60)
+    logger = CustomLogger("job_detail.json")
+    logger.update_dut_time("start", None)
+    poe = PoERun(args, args.test_timeout * 60, logger)
     retval = poe.run()
 
     poe.logged_system(args.powerdown)
+    logger.update_dut_time("end", None)
 
     sys.exit(retval)
 

.gitlab-ci/container/debian/arm_test.sh

@@ -17,6 +17,8 @@ DEPS=(
     openssh-server
     procps
     python3-distutils
+    python3-filelock
+    python3-fire
     python3-minimal
     python3-serial
     rsync

.gitlab-ci/image-tags.yml

@@ -18,14 +18,14 @@ variables:
    DEBIAN_X86_64_TEST_IMAGE_VK_PATH: "debian/x86_64_test-vk"
    DEBIAN_X86_64_TEST_ANDROID_IMAGE_PATH: "debian/x86_64_test-android"
 
-   DEBIAN_X86_64_TEST_ANDROID_TAG: "2023-11-18-deqp-runner"
+   DEBIAN_X86_64_TEST_ANDROID_TAG: "2023-12-14-structured-logs"
-   DEBIAN_X86_64_TEST_GL_TAG: "2023-11-06-deqp-fixes"
+   DEBIAN_X86_64_TEST_GL_TAG: "2023-12-14-structured-logs"
-   DEBIAN_X86_64_TEST_VK_TAG: "2023-12-08-vkd3d"
+   DEBIAN_X86_64_TEST_VK_TAG: "2023-12-14-structured-logs"
 
    ALPINE_X86_64_BUILD_TAG: "2023-10-30-ci-improv"
    ALPINE_X86_64_LAVA_SSH_TAG: "2023-06-26-first-version"
    FEDORA_X86_64_BUILD_TAG: "2023-12-11-meson-bump"
-   KERNEL_ROOTFS_TAG: "2023-12-08-firmware"
+   KERNEL_ROOTFS_TAG: "2023-12-14-structured-logs"
    KERNEL_TAG: "v6.6.4-for-mesa-ci-e4f4c500f7fb"
    KERNEL_REPO: "gfx-ci/linux"
 

.gitlab-ci/test/gitlab-ci.yml

@@ -22,7 +22,7 @@
 .formatting-check:
   # Cancel job if a newer commit is pushed to the same branch
   interruptible: true
-  stage: lint
+  stage: code-validation
   extends:
     - .use-debian/x86_64_build
   variables:
@@ -57,6 +57,29 @@ clang-format:
     - clang-format-${LLVM_VERSION} --version
     - ninja -C build clang-format
 
+.test-check:
+  # Cancel job if a newer commit is pushed to the same branch
+  interruptible: true
+  stage: code-validation
+  extends:
+    - .use-debian/x86_64_build
+  variables:
+    GIT_STRATEGY: fetch
+  timeout: 10m
+
+python-test:
+  extends:
+    - .test-check
+  script:
+    - cd bin/ci
+    - pip install --break-system-packages -r test/requirements.txt
+    - PYTHONPATH=. pytest -v
+  rules:
+    - !reference [.disable-farm-mr-rules, rules]
+    - !reference [.never-post-merge-rules, rules]
+    - changes:
+        - bin/ci/**/*
+
 .test-gl:
   extends:
     - .test

bin/ci/test/requirements.txt

@@ -0,0 +1,5 @@
+filelock==3.12.4
+fire==0.5.0
+mock==5.1.0
+polars==0.19.3
+pytest==7.4.2

src/broadcom/ci/gitlab-ci-inc.yml

@@ -107,6 +107,7 @@
   variables:
     HWCI_TEST_SCRIPT: "/install/deqp-runner.sh"
     FLAKES_CHANNEL: "#videocore-ci"
+    FARM: igalia
   timeout: 20m
 
 .broadcom-test:arm64:

src/broadcom/ci/gitlab-ci.yml

@@ -14,6 +14,7 @@ vc4-rpi3-gl:arm32:
     HWCI_KERNEL_MODULES: vc4
     DEQP_SUITE: broadcom-rpi3
     HWCI_START_WESTON: 1
+    DEVICE_TYPE: rpi3
 
 vc4-rpi3-gl-piglit-full:arm32:
   extends:
@@ -24,6 +25,7 @@ vc4-rpi3-gl-piglit-full:arm32:
     - igalia-fullrun
   variables:
     DEQP_SUITE: broadcom-rpi3-piglit-full
+    DEVICE_TYPE: rpi3
 
 
 v3d-rpi4-gl:arm64:
@@ -39,6 +41,7 @@ v3d-rpi4-gl:arm64:
     HWCI_START_WESTON: 1
     DEQP_SUITE: broadcom-rpi4
     DEQP_FRACTION: 2
+    DEVICE_TYPE: rpi4
 
 v3d-rpi4-gl-full:arm64:
   extends:
@@ -52,6 +55,7 @@ v3d-rpi4-gl-full:arm64:
   variables:
     TEST_PHASE_TIMEOUT: 40
     DEQP_FRACTION: 1
+    DEVICE_TYPE: rpi4
 
 
 v3d-rpi4-traces:arm64:
@@ -66,6 +70,7 @@ v3d-rpi4-traces:arm64:
     PIGLIT_TRACES_FILE: traces-broadcom.yml
     PIGLIT_REPLAY_DEVICE_NAME: "broadcom-rpi4"
     PIGLIT_RESULTS: "broadcom-rpi4-replay"
+    DEVICE_TYPE: rpi4
   tags:
     - igalia-rpi4
 
@@ -84,6 +89,7 @@ v3dv-rpi4-vk:arm64:
     GPU_VERSION: broadcom-rpi4
     HWCI_KERNEL_MODULES: v3d,vc4
     VK_DRIVER: broadcom
+    DEVICE_TYPE: rpi4
   tags:
     - igalia-rpi4
 
@@ -99,3 +105,4 @@ v3dv-rpi4-vk-full:arm64:
   variables:
     TEST_PHASE_TIMEOUT: 80
     DEQP_FRACTION: 1
+    DEVICE_TYPE: rpi4

src/gallium/drivers/nouveau/ci/gitlab-ci.yml

@@ -10,3 +10,5 @@ gk20a-gles:
   variables:
     HWCI_TEST_SCRIPT: "/install/deqp-runner.sh"
     DEQP_SUITE: nouveau-gk20a
+    FARM : anholt
+    DEVICE_TYPE: anholt-jetson