Docker build in CI

.gitlab-ci.yml

@@ -0,0 +1,110 @@
+stages:
+  - build
+  - publish
+  - generate
+  - trigger
+  - test
+
+# Build commit SHA image
+build-image:
+  stage: build
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
+  variables:
+    IMAGE_SHA: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  script:
+    # Auth for Kaniko
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+    # Build & push SHA-tagged image
+    - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $IMAGE_SHA
+    - echo "Image pushed as $IMAGE_SHA"
+    # leave some trace that the image was built
+    - echo '$IMAGE_SHA' | tee build-image.txt
+  rules:
+    - changes:
+        - Dockerfile
+        - dev_requirements.txt
+  artifacts:
+    paths:
+      - build-image.txt
+
+
+.publish-image:
+  stage: publish
+  image: quay.io/skopeo/stable
+  script:
+    - export SRC="$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA"
+    - export DEST="$CI_REGISTRY_IMAGE:$IMAGE_VERSION"
+    - |
+      skopeo copy \
+        --src-creds "$CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD" \
+        --dest-creds "$CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD" \
+        docker://"$SRC" docker://"$DEST"
+  needs:
+    - job: build-image
+      optional: true
+
+publish-image-latest:
+  extends: .publish-image
+  variables:
+    IMAGE_VERSION: latest
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+      changes:
+        - Dockerfile
+        - dev_requirements.txt
+    - if: $CI_COMMIT_TAG
+
+publish-image-tag:
+  extends: .publish-image
+  variables:
+    IMAGE_VERSION: $CI_COMMIT_TAG
+  rules:
+    - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+$/'
+
+# set the image tag to use based on whether build-image ran or not
+# - build-image ran: use the commit SHA
+# - build-image didn't run: use "latest"
+set-image-tag:
+  stage: publish
+  script:
+    - if [[ -s "build-image.txt" ]]; then
+        echo "IMAGE_TAG=$CI_COMMIT_SHA" | tee image.env;
+      else
+        echo "IMAGE_TAG=latest" | tee image.env;
+      fi
+  artifacts:
+    reports:
+      dotenv: image.env
+  rules:
+    - when: always
+  needs:
+    - job: build-image
+      optional: true
+
+generator:
+  stage: generate
+  image: python:3.8-alpine
+  tags:
+    - k8s-default
+  before_script:
+    - pip install pyyaml
+  script:
+    - cd tests
+    - python analyze_tests.py
+  artifacts:
+    paths:
+      - tests/pytests.yml
+
+pytests:
+  stage: trigger
+  trigger:
+    include:
+      - local: tests/ci.yml
+      - artifact: tests/pytests.yml
+        job: generator
+    strategy: depend
+  variables:
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+

Dockerfile

@@ -0,0 +1,27 @@
+FROM ghdl/ghdl:6.0.0-dev-gcc-ubuntu-22.04
+LABEL [email protected]
+RUN apt update && \
+    apt install -y build-essential wget git ca-certificates && \
+    useradd --create-home --shell /bin/bash conifer
+# Add Tini
+ENV TINI_VERSION=v0.19.0
+ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
+RUN chmod +x /tini
+ENTRYPOINT ["/tini", "--"]
+USER conifer
+ENV WORKDIR=/home/conifer
+WORKDIR $WORKDIR
+COPY . .
+RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \
+    bash Miniforge3.sh -b -p "${HOME}/conda" && \
+    source "${HOME}/conda/etc/profile.d/mamba.sh" && \
+    mamba activate && \
+    mamba shell init && \
+    pip install -r dev_requirements.txt && \
+    git clone --depth 1 --branch v3.12.0 https://github.com/nlohmann/json.git && \
+    git clone --depth 1 https://github.com/Xilinx/HLS_arbitrary_Precision_Types.git && \
+    pip install .
+ENV JSON_ROOT=${WORKDIR}/json/single_include
+ENV XILINX_AP_INCLUDE=${WORKDIR}/HLS_arbitrary_Precision_Types/include
+ENV PATH="${WORKDIR}/conda/bin:${PATH}"
+CMD ["/bin/bash"]

dev_requirements.txt

@@ -3,7 +3,8 @@ onnxruntime
 scipy
 onnxmltools
 scikit-learn
-xgboost<2.0.0
-pybind11
+skl2onnx
+xgboost<3.0.0
+pybind11<3.0.0
 ydf
 pandas

tests/analyze_tests.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+import yaml
+
+template = '''
+pytest.{name}:
+  extends: .pytest-{extends}
+  variables:
+    PYTESTFILE: {test_file}
+  allow_failure: {allow_failure}
+'''
+
+# override the auto detection of which script to extend for the following jobs
+extends_override = {'backends' : 'fpga'}
+
+# allow the following jobs to fail
+allow_failure = ['backends', 'xgb_converter', 'onnx_to_hls']
+
+# check whether "build" method is called in the test -> needs different resources
+def calls_build(test_filename):
+    with open(test_filename) as f:
+        content = f.read()
+        return '.build(' in content
+
+def generate_test_yaml(directory='.'):
+    # List of test files to scan
+    test_dir = Path(directory)
+    test_files = list(test_dir.glob("test_*.py"))
+
+    yml = None
+
+    for test_file in test_files:
+        file_name = str(test_file)
+        name = file_name.replace('test_', '').replace('.py', '')
+        build = calls_build(test_file)
+        extends = 'fpga' if build else 'plain'
+        if name in extends_override.keys():
+            extends = extends_override[name]
+        allow_fail = 'True' if name in allow_failure else 'False'
+        test_yml = yaml.safe_load(template.format(name=name,
+                                                  extends=extends,
+                                                  test_file=test_file,
+                                                  allow_failure=allow_fail))
+        if yml is None:
+            yml = test_yml
+        else:
+            yml.update(test_yml)
+
+    return yml
+
+if __name__ == '__main__':
+    yml = generate_test_yaml(Path(__file__).parent)
+    with open('pytests.yml', 'w') as yamlfile:
+        yaml.safe_dump(yml, yamlfile)

tests/ci.yml

@@ -0,0 +1,42 @@
+.snippets:
+  before_script:
+    - git clone --depth 1 --branch v3.12.0 https://github.com/nlohmann/json.git
+    - export JSON_ROOT=$(pwd)/json/single_include/
+    - git clone --depth 1 https://github.com/Xilinx/HLS_arbitrary_Precision_Types.git
+    - export XILINX_AP_INCLUDE=$(pwd)/HLS_arbitrary_Precision_Types/include
+    - pip install -r dev_requirements.txt
+  conifer_reinstall:
+    - pip uninstall -y conifer
+    - pip install .
+    - pip show conifer
+
+.pytest:
+  stage: test
+  script:
+    - !reference [.snippets, conifer_reinstall]
+    - cd tests
+    - pytest $PYTESTFILE -rA
+  artifacts:
+    when: always
+    paths:
+      - tests/prj*
+  rules:
+    - when: always
+
+# the $IMAGE_TAG is provided by set-image-tag job
+.pytest-plain:
+  extends: .pytest
+  image: $CI_REGISTRY_IMAGE:$IMAGE_TAG
+  needs:
+    - pipeline: $PARENT_PIPELINE_ID
+      job: set-image-tag
+  tags:
+    - k8s-default
+
+.pytest-fpga:
+  extends: .pytest
+  image: registry.cern.ch/ci4fpga/vivado:2024.1
+  tags:
+    - fpga-mid
+  before_script:
+    - !reference [.snippets, before_script]

tests/conftest.py

@@ -1,2 +1,9 @@
 import pytest
-from tests.util import train_skl, hls_convert, vhdl_convert, predict
+import logging
+from tests.util import train_skl, hls_convert, vhdl_convert, predict
+
+@pytest.fixture(autouse=True)
+def no_logs_gte_error(caplog):
+    yield
+    errors = [record for record in caplog.get_records('call') if record.levelno >= logging.ERROR]
+    assert not errors