diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..851e3a2 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,110 @@ +stages: + - build + - publish + - generate + - trigger + - test + +# Build commit SHA image +build-image: + stage: build + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + variables: + IMAGE_SHA: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA + script: + # Auth for Kaniko + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + # Build & push SHA-tagged image + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $IMAGE_SHA + - echo "Image pushed as $IMAGE_SHA" + # leave some trace that the image was built + - echo '$IMAGE_SHA' | tee build-image.txt + rules: + - changes: + - Dockerfile + - dev_requirements.txt + artifacts: + paths: + - build-image.txt + + +.publish-image: + stage: publish + image: quay.io/skopeo/stable + script: + - export SRC="$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA" + - export DEST="$CI_REGISTRY_IMAGE:$IMAGE_VERSION" + - | + skopeo copy \ + --src-creds "$CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD" \ + --dest-creds "$CI_REGISTRY_USER:$CI_REGISTRY_PASSWORD" \ + docker://"$SRC" docker://"$DEST" + needs: + - job: build-image + optional: true + +publish-image-latest: + extends: .publish-image + variables: + IMAGE_VERSION: latest + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + changes: + - Dockerfile + - dev_requirements.txt + - if: $CI_COMMIT_TAG + +publish-image-tag: + extends: .publish-image + variables: + IMAGE_VERSION: $CI_COMMIT_TAG + rules: + - if: '$CI_COMMIT_TAG =~ /^v[0-9]+\.[0-9]+$/' + +# set the image tag to use based on whether build-image ran or not +# - build-image ran: use the commit SHA +# - build-image didn't run: use "latest" +set-image-tag: + stage: publish + script: + - if [[ -s "build-image.txt" ]]; then + echo "IMAGE_TAG=$CI_COMMIT_SHA" | tee image.env; + else + echo "IMAGE_TAG=latest" | tee image.env; + fi + artifacts: + reports: + dotenv: image.env + rules: + - when: always + needs: + - job: build-image + optional: true + +generator: + stage: generate + image: python:3.8-alpine + tags: + - k8s-default + before_script: + - pip install pyyaml + script: + - cd tests + - python analyze_tests.py + artifacts: + paths: + - tests/pytests.yml + +pytests: + stage: trigger + trigger: + include: + - local: tests/ci.yml + - artifact: tests/pytests.yml + job: generator + strategy: depend + variables: + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d1d397f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM ghdl/ghdl:6.0.0-dev-gcc-ubuntu-22.04 +LABEL maintainer=sioni@cern.ch +RUN apt update && \ + apt install -y build-essential wget git ca-certificates && \ + useradd --create-home --shell /bin/bash conifer +# Add Tini +ENV TINI_VERSION=v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] +USER conifer +ENV WORKDIR=/home/conifer +WORKDIR $WORKDIR +COPY . . +RUN wget -O Miniforge3.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-$(uname -m).sh" && \ + bash Miniforge3.sh -b -p "${HOME}/conda" && \ + source "${HOME}/conda/etc/profile.d/mamba.sh" && \ + mamba activate && \ + mamba shell init && \ + pip install -r dev_requirements.txt && \ + git clone --depth 1 --branch v3.12.0 https://github.com/nlohmann/json.git && \ + git clone --depth 1 https://github.com/Xilinx/HLS_arbitrary_Precision_Types.git && \ + pip install . +ENV JSON_ROOT=${WORKDIR}/json/single_include +ENV XILINX_AP_INCLUDE=${WORKDIR}/HLS_arbitrary_Precision_Types/include +ENV PATH="${WORKDIR}/conda/bin:${PATH}" +CMD ["/bin/bash"] \ No newline at end of file diff --git a/dev_requirements.txt b/dev_requirements.txt index 00c2785..4a2cff9 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -3,7 +3,8 @@ onnxruntime scipy onnxmltools scikit-learn -xgboost<2.0.0 -pybind11 +skl2onnx +xgboost<3.0.0 +pybind11<3.0.0 ydf pandas diff --git a/tests/analyze_tests.py b/tests/analyze_tests.py new file mode 100644 index 0000000..fc9f11d --- /dev/null +++ b/tests/analyze_tests.py @@ -0,0 +1,53 @@ +from pathlib import Path +import yaml + +template = ''' +pytest.{name}: + extends: .pytest-{extends} + variables: + PYTESTFILE: {test_file} + allow_failure: {allow_failure} +''' + +# override the auto detection of which script to extend for the following jobs +extends_override = {'backends' : 'fpga'} + +# allow the following jobs to fail +allow_failure = ['backends', 'xgb_converter', 'onnx_to_hls'] + +# check whether "build" method is called in the test -> needs different resources +def calls_build(test_filename): + with open(test_filename) as f: + content = f.read() + return '.build(' in content + +def generate_test_yaml(directory='.'): + # List of test files to scan + test_dir = Path(directory) + test_files = list(test_dir.glob("test_*.py")) + + yml = None + + for test_file in test_files: + file_name = str(test_file) + name = file_name.replace('test_', '').replace('.py', '') + build = calls_build(test_file) + extends = 'fpga' if build else 'plain' + if name in extends_override.keys(): + extends = extends_override[name] + allow_fail = 'True' if name in allow_failure else 'False' + test_yml = yaml.safe_load(template.format(name=name, + extends=extends, + test_file=test_file, + allow_failure=allow_fail)) + if yml is None: + yml = test_yml + else: + yml.update(test_yml) + + return yml + +if __name__ == '__main__': + yml = generate_test_yaml(Path(__file__).parent) + with open('pytests.yml', 'w') as yamlfile: + yaml.safe_dump(yml, yamlfile) \ No newline at end of file diff --git a/tests/ci.yml b/tests/ci.yml new file mode 100644 index 0000000..52b5cf3 --- /dev/null +++ b/tests/ci.yml @@ -0,0 +1,42 @@ +.snippets: + before_script: + - git clone --depth 1 --branch v3.12.0 https://github.com/nlohmann/json.git + - export JSON_ROOT=$(pwd)/json/single_include/ + - git clone --depth 1 https://github.com/Xilinx/HLS_arbitrary_Precision_Types.git + - export XILINX_AP_INCLUDE=$(pwd)/HLS_arbitrary_Precision_Types/include + - pip install -r dev_requirements.txt + conifer_reinstall: + - pip uninstall -y conifer + - pip install . + - pip show conifer + +.pytest: + stage: test + script: + - !reference [.snippets, conifer_reinstall] + - cd tests + - pytest $PYTESTFILE -rA + artifacts: + when: always + paths: + - tests/prj* + rules: + - when: always + +# the $IMAGE_TAG is provided by set-image-tag job +.pytest-plain: + extends: .pytest + image: $CI_REGISTRY_IMAGE:$IMAGE_TAG + needs: + - pipeline: $PARENT_PIPELINE_ID + job: set-image-tag + tags: + - k8s-default + +.pytest-fpga: + extends: .pytest + image: registry.cern.ch/ci4fpga/vivado:2024.1 + tags: + - fpga-mid + before_script: + - !reference [.snippets, before_script] \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index bb67fec..aacd386 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,2 +1,9 @@ import pytest -from tests.util import train_skl, hls_convert, vhdl_convert, predict \ No newline at end of file +import logging +from tests.util import train_skl, hls_convert, vhdl_convert, predict + +@pytest.fixture(autouse=True) +def no_logs_gte_error(caplog): + yield + errors = [record for record in caplog.get_records('call') if record.levelno >= logging.ERROR] + assert not errors \ No newline at end of file