diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 412fb9b..ff1d3d3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,21 +2,28 @@ # Additionally, lint the code before anything else to fail more quickly stages: - lint + - check - build - test - release - dockerify default: - image: "rust:latest" + image: "registry.gitlab.com/fabinfra/rust-builder:latest" tags: - linux - docker + - fabinfra variables: GIT_SUBMODULE_STRATEGY: recursive CARGO_HOME: $CI_PROJECT_DIR/cargo APT_CACHE_DIR: $CI_PROJECT_DIR/apt + FF_USE_FASTZIP: "true" # enable fastzip - a faster zip implementation that also supports level configuration. + ARTIFACT_COMPRESSION_LEVEL: fast # can also be set to fastest, fast, slow and slowest. If just enabling fastzip is not enough try setting this to fastest or fast. + CACHE_COMPRESSION_LEVEL: fastest # same as above, but for caches + TRANSFER_METER_FREQUENCY: 5s # will display transfer progress every 5 seconds for artifacts and remote caches. + # cache dependencies and build environment to speed up setup cache: @@ -26,10 +33,6 @@ cache: - cargo/ - target/ -# install build dependencies -before_script: - - apt-get update -yqq - - apt-get install -o dir::cache::archives="$APT_CACHE_DIR" -yqq --no-install-recommends capnproto build-essential cmake clang libclang-dev jq .lints: stage: lint @@ -41,7 +44,6 @@ before_script: lint:clippy: extends: .lints script: - - rustup component add clippy - cargo clippy -V - echo -e "\e[0Ksection_start:`date +%s`:clippy_output\r\e[0Kcargo clippy output" - cargo clippy -- --no-deps @@ -51,15 +53,14 @@ lint:clippy: lint:fmt: extends: .lints script: - - rustup component add rustfmt - cargo fmt --version - echo -e "\e[0Ksection_start:`date +%s`:rustfmt_output\r\e[0KChanges suggested by rustfmt" - cargo fmt --check -- -v - echo -e "\e[0Ksection_end:`date +%s`:rustfmt_output\r\e[0K" # Check if the code builds on rust stable -stable:build: - stage: build +stable:check: + stage: check only: - main - development @@ -70,9 +71,94 @@ stable:build: - cargo check --verbose - echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K" +# Check if the code builds on rust stable on armv7 +stable:check:armhf: + stage: check + only: + - main + - development + - merge_requests + before_script: + - mkdir -p $CARGO_HOME + - cp cargo-cross-config $CARGO_HOME/config.toml + script: + - rustc +stable --version && cargo --version + - echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo check with target armv7-unknown-linux-gnueabihf" + - cargo check --verbose --target armv7-unknown-linux-gnueabihf + - echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K" + + # Check if the code builds on rust stable on arm64 +stable:check:arm64: + stage: check + only: + - main + - development + - merge_requests + before_script: + - mkdir -p $CARGO_HOME + - cp cargo-cross-config $CARGO_HOME/config.toml + script: + - rustc +stable --version && cargo --version + - echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo check with target aarch64-unknown-linux-gnu" + - cargo check --verbose --target aarch64-unknown-linux-gnu + - echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K" + +# Check if the code builds on rust stable +stable:build:amd64: + stage: build + only: + - main + - development + - merge_requests + script: + - rustc +stable --version && cargo --version + - echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo build with target x86_64-unknown-linux-gnu" + - cargo build --release --target x86_64-unknown-linux-gnu + - echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K" + artifacts: + paths: + - target/x86_64-unknown-linux-gnu/release/bffhd + + +# Check if the code builds on rust stable on armv7 +stable:build:armhf: + stage: build + only: + - main + - development + before_script: + - mkdir -p $CARGO_HOME + - cp cargo-cross-config $CARGO_HOME/config.toml + script: + - rustc +stable --version && cargo --version + - echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo build with target armv7-unknown-linux-gnueabihf" + - cargo build --release --target armv7-unknown-linux-gnueabihf + - echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K" + artifacts: + paths: + - target/armv7-unknown-linux-gnueabihf/release/bffhd + + # Check if the code builds on rust stable on arm64 +stable:build:arm64: + stage: build + only: + - main + - development + before_script: + - mkdir -p $CARGO_HOME + - cp cargo-cross-config $CARGO_HOME/config.toml + script: + - rustc +stable --version && cargo --version + - echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo build with target aarch64-unknown-linux-gnu" + - cargo build --release --target aarch64-unknown-linux-gnu + - echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K" + artifacts: + paths: + - target/aarch64-unknown-linux-gnu/release/bffhd + stable:test: stage: build - needs: ["stable:build"] + needs: ["stable:check"] only: - main - development @@ -80,14 +166,12 @@ stable:test: script: - echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo test --no-run" - cargo test --verbose --no-run --workspace - - echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K" - - cargo install --root $CARGO_HOME cargo2junit .tests: stage: test needs: ["stable:test"] script: - - cargo test --workspace $TEST_TARGET -- -Z unstable-options --format json --report-time | $CARGO_HOME/bin/cargo2junit > report.xml + - cargo test --workspace $TEST_TARGET -- -Z unstable-options --format json --report-time | cargo2junit > report.xml artifacts: when: always reports: @@ -114,6 +198,23 @@ unit test 3:3: TEST_TARGET: "--examples" extends: .tests +upload_binaries: + stage: release + image: curlimages/curl:latest + before_script: [] + cache: [] + dependencies: + - stable:build:amd64 + - stable:build:armhf + - stable:build:arm64 + script: + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file target/aarch64-unknown-linux-gnu/release/bffhd "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${CI_COMMIT_TAG}/bffhd_${VERSION}_linux_arm64"' + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file target/x86_64-unknown-linux-gnu/release/bffhd "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${CI_COMMIT_TAG}/bffhd_${VERSION}_linux_amd64"' + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file target/armv7-unknown-linux-gnueabihf/release/bffhd "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${VERSION}/bffhd_${VERSION}_linux_arm"' + rules: + - if: $CI_COMMIT_TAG =~ "release/.*" + when: never + - if: $CI_COMMIT_BRANCH == "main" release_prepare: stage: release @@ -144,32 +245,106 @@ release_job: name: "BFFH $VERSION" description: "GitLab CI auto-created release" tag_name: "release/$VERSION" + assets: + links: + - name: 'bffhd AMD64' + url: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${VERSION}/bffhd_${VERSION}_linux_amd64" + - name: 'bffhd ARMv7' + url: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${VERSION}/bffhd_${VERSION}_linux_arm" + - name: 'bffhd ARM64' + url: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${VERSION}/bffhd_${VERSION}_linux_arm64" build:docker-releases: stage: dockerify - image: - name: gcr.io/kaniko-project/executor:v1.6.0-debug - entrypoint: [""] + image: jdrouet/docker-with-buildx:latest + dependencies: + - stable:build:amd64 + - stable:build:armhf + - stable:build:arm64 + tags: + - linux + - docker + - fabinfra + variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + TRIVY_NO_PROGRESS: "true" + TRIVY_CACHE_DIR: ".trivycache/" + services: + - docker:dind before_script: - - '' + - export TRIVY_VERSION=$(wget -qO - "https://api.github.com/repos/aquasecurity/trivy/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/') + - echo $TRIVY_VERSION + - wget --no-verbose https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz -O - | tar -zxvf - script: - - mkdir -p /kaniko/.docker - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - - /kaniko/executor --force --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG + - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - docker buildx create --name cibuilder --driver docker-container --use + - docker buildx ls + - docker buildx inspect --bootstrap + - docker buildx build --platform linux/arm/v7,linux/arm64,linux/amd64 -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG . + - docker buildx build --load --platform linux/amd64 -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG . + # Build report + - ./trivy image --exit-code 0 --format template --template "@contrib/gitlab.tpl" -o gl-container-scanning-report.json $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG + # Print report + - ./trivy image --exit-code 0 --severity HIGH $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG + # Fail on severe vulnerabilities + - ./trivy image --exit-code 1 --severity CRITICAL $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG + cache: + paths: + - .trivycache/ + artifacts: + reports: + container_scanning: gl-container-scanning-report.json rules: - if: $CI_COMMIT_TAG =~ "release/.*" when: never build:docker-development: stage: dockerify - image: - name: gcr.io/kaniko-project/executor:v1.6.0-debug - entrypoint: [""] + image: jdrouet/docker-with-buildx:latest + dependencies: + - stable:build:amd64 + - stable:build:armhf + - stable:build:arm64 + tags: + - linux + - docker + - fabinfra + variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + TRIVY_NO_PROGRESS: "true" + TRIVY_CACHE_DIR: ".trivycache/" + services: + - docker:dind before_script: - - '' + - export TRIVY_VERSION=$(wget -qO - "https://api.github.com/repos/aquasecurity/trivy/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/') + - echo $TRIVY_VERSION + - wget --no-verbose https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz -O - | tar -zxvf - script: - - mkdir -p /kaniko/.docker - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - - /kaniko/executor --force --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:dev-latest + - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - docker buildx create --name cibuilder --driver docker-container --use + - docker buildx ls + - docker buildx inspect --bootstrap + - docker buildx build --platform linux/arm/v7,linux/arm64,linux/amd64 -t $CI_REGISTRY_IMAGE:development . + - docker buildx build --load --platform linux/amd64 -t $CI_REGISTRY_IMAGE:development . + # Build report + - ./trivy image --exit-code 0 --format template --template "@contrib/gitlab.tpl" -o gl-container-scanning-report.json $CI_REGISTRY_IMAGE:development + # Print report + - ./trivy image --exit-code 0 --severity HIGH $CI_REGISTRY_IMAGE:development + # Fail on severe vulnerabilities + - ./trivy image --exit-code 1 --severity CRITICAL $CI_REGISTRY_IMAGE:development + - docker push $CI_REGISTRY_IMAGE:development + cache: + paths: + - .trivycache/ + artifacts: + reports: + container_scanning: gl-container-scanning-report.json only: - development diff --git a/Dockerfile b/Dockerfile index 0732c9c..418c22e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,23 @@ -# Setup build image for multistage build -FROM rust:bullseye as builder -# install build deps -RUN apt-get update && apt-get upgrade -y -RUN apt-get install -yqq --no-install-recommends capnproto python3 python3-pip -RUN pip3 install paho-mqtt +FROM --platform=$BUILDPLATFORM alpine:latest as copy +ARG TARGETPLATFORM +RUN case "$TARGETPLATFORM" in \ + "linux/arm/v7") echo armv7-unknown-linux-gnueabihf > /rust_target.txt ;; \ + "linux/arm/v6") echo arm-unknown-linux-gnueabihf > /rust_target.txt ;; \ + "linux/arm64") echo aarch64-unknown-linux-gnu > /rust_target.txt ;; \ + "linux/amd64") echo x86_64-unknown-linux-gnu > /rust_target.txt ;; \ + *) exit 1 ;; \ +esac WORKDIR /usr/src/bffh COPY . . -RUN cargo build --release - +RUN cp target/$(cat /rust_target.txt)/release/bffhd ./bffhd.bin # Setup deployable image -FROM debian:bullseye-slim -# Install runtime deps -#RUN apt-get update && apt-get upgrade -yqq -COPY --from=builder /usr/src/bffh/target/release/bffhd /usr/local/bin/bffhd -#COPY --from=builder /usr/src/bffh/examples/bffh.dhall /etc/diflouroborane.dhall -# RUN diflouroborane --print-default > /etc/diflouroborane.toml +FROM ubuntu:22.04 +RUN apt-get update && apt-get upgrade -y +RUN apt-get install -yqq --no-install-recommends python3 python3-pip +RUN pip3 install paho-mqtt +COPY --from=copy /usr/src/bffh/bffhd.bin /usr/local/bin/bffhd VOLUME /etc/bffh/ VOLUME /var/lib/bffh/ VOLUME /usr/local/lib/bffh/adapters/ diff --git a/bffhd/resources/state/value.rs b/bffhd/resources/state/value.rs index e87fdde..9b62e2c 100644 --- a/bffhd/resources/state/value.rs +++ b/bffhd/resources/state/value.rs @@ -275,10 +275,6 @@ pub struct ImplDebugInfo { /// [statevalue_register](macro@crate::statevalue_register) macro with your OID as first and type /// as second parameter like so: /// -/// ```no_run -/// struct MyStruct; -/// statevalue_register!(ObjectIdentifier::from_str("1.3.6.1.4.1.48398.612.1.14").unwrap(), MyStruct) -/// ``` pub struct ImplEntry<'a> { id: ImplId<'a>, data: ImplData<'a>, diff --git a/cargo-cross-config b/cargo-cross-config new file mode 100644 index 0000000..ffbdcaa --- /dev/null +++ b/cargo-cross-config @@ -0,0 +1,8 @@ +[target.armv7-unknown-linux-gnueabihf] +linker = "arm-linux-gnueabihf-gcc" + +[target.arm-unknown-linux-gnueabihf] +linker = "arm-linux-gnueabi-gcc" + +[target.aarch64-unknown-linux-gnu] +linker = "aarch64-linux-gnu-gcc" diff --git a/runtime/lightproc/examples/proc_panic.rs b/runtime/lightproc/examples/proc_panic.rs index 4fc0c20..3db742e 100644 --- a/runtime/lightproc/examples/proc_panic.rs +++ b/runtime/lightproc/examples/proc_panic.rs @@ -30,7 +30,8 @@ where } let schedule = |t| (QUEUE.deref()).send(t).unwrap(); - let (proc, handle) = LightProc::recoverable(future, schedule); + let span = tracing::trace_span!("runtime.spawn", kind = "local"); + let (proc, handle) = LightProc::recoverable(future, schedule, span, None); let handle = handle.on_panic( |err: Box| match err.downcast::<&'static str>() { diff --git a/runtime/lightproc/examples/proc_run.rs b/runtime/lightproc/examples/proc_run.rs index 483a183..bdeb28f 100644 --- a/runtime/lightproc/examples/proc_run.rs +++ b/runtime/lightproc/examples/proc_run.rs @@ -17,7 +17,8 @@ where let future = async move { fut.await }; let schedule = move |t| sender.send(t).unwrap(); - let (proc, handle) = LightProc::build(future, schedule); + let span = tracing::trace_span!("runtime.spawn", kind = "local"); + let (proc, handle) = LightProc::build(future, schedule, span, None); proc.schedule(); diff --git a/runtime/lightproc/src/lightproc.rs b/runtime/lightproc/src/lightproc.rs index 63af3cd..cc0cf25 100644 --- a/runtime/lightproc/src/lightproc.rs +++ b/runtime/lightproc/src/lightproc.rs @@ -9,6 +9,7 @@ //! # Example Usage //! //! ```rust +//! use tracing::Span; //! use lightproc::prelude::*; //! //! // ... future that does work @@ -23,6 +24,8 @@ //! let panic_recoverable = LightProc::recoverable( //! future, //! schedule_function, +//! Span::current(), +//! None, //! ); //! ``` @@ -60,6 +63,7 @@ impl LightProc { /// # Example /// ```rust /// # use std::any::Any; + /// # use tracing::Span; /// # use lightproc::prelude::*; /// # /// # // ... basic schedule function with no waker logic @@ -72,9 +76,11 @@ impl LightProc { /// let (proc, handle) = LightProc::recoverable( /// future, /// schedule_function, + /// Span::current(), + /// None /// ); - /// let handle = handle.on_panic(|s: &mut EmptyProcState, e: Box| { - /// let reason = e.downcast::(); + /// let handle = handle.on_panic(|e: Box| { + /// let reason = e.downcast::().unwrap(); /// println!("future panicked!: {}", &reason); /// }); /// ``` @@ -110,13 +116,6 @@ impl LightProc { /// # // ... basic schedule function with no waker logic /// # fn schedule_function(proc: LightProc) {;} /// # - /// # // ... process stack with a lifecycle callback - /// # let proc_stack = - /// # ProcStack::default() - /// # .with_after_panic(|s: &mut EmptyProcState| { - /// # println!("After panic started!"); - /// # }); - /// # /// // ... creating a standard process /// let standard = LightProc::build( /// future, diff --git a/runtime/lightproc/src/recoverable_handle.rs b/runtime/lightproc/src/recoverable_handle.rs index eb5bc10..d1be89e 100644 --- a/runtime/lightproc/src/recoverable_handle.rs +++ b/runtime/lightproc/src/recoverable_handle.rs @@ -49,8 +49,7 @@ impl RecoverableHandle { /// /// ```rust /// # use std::any::Any; - /// use lightproc::proc_stack::ProcStack; - /// use lightproc::proc_state::EmptyProcState; + /// # use tracing::Span; /// # use lightproc::prelude::*; /// # /// # // ... future that does work @@ -61,21 +60,16 @@ impl RecoverableHandle { /// # // ... basic schedule function with no waker logic /// # fn schedule_function(proc: LightProc) {;} /// # - /// # // ... process stack with a lifecycle callback - /// # let proc_stack = - /// # ProcStack::default() - /// # .with_after_panic(|s: &mut EmptyProcState| { - /// # println!("After panic started!"); - /// # }); - /// # /// // ... creating a recoverable process /// let (proc, recoverable) = LightProc::recoverable( /// future, /// schedule_function, + /// Span::current(), + /// None /// ); /// /// recoverable - /// .on_return(|_e: Box| { + /// .on_panic(|_e: Box| { /// println!("Inner future panicked"); /// }); /// ```