vmario
/
fabaccess-bffh
mirror of https://gitlab.com/fabinfra/fabaccess/bffh.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

reworked CI

This commit is contained in:
Kai Kriegel 2023-01-02 03:59:09 +00:00
parent 0d2cd6f376
commit 0380e02f3f
8 changed files with 241 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

229
.gitlab-ci.yml
View File

@ -2,21 +2,28 @@
# Additionally, lint the code before anything else to fail more quickly # Additionally, lint the code before anything else to fail more quickly
stages: stages:
- lint - lint
- check
- build - build
- test - test
- release - release
- dockerify - dockerify
default: default:
image: "rust:latest" image: "registry.gitlab.com/fabinfra/rust-builder:latest"
tags: tags:
- linux - linux
- docker - docker
- fabinfra
variables: variables:
GIT_SUBMODULE_STRATEGY: recursive GIT_SUBMODULE_STRATEGY: recursive
CARGO_HOME: $CI_PROJECT_DIR/cargo CARGO_HOME: $CI_PROJECT_DIR/cargo
APT_CACHE_DIR: $CI_PROJECT_DIR/apt APT_CACHE_DIR: $CI_PROJECT_DIR/apt
FF_USE_FASTZIP: "true" # enable fastzip - a faster zip implementation that also supports level configuration.
ARTIFACT_COMPRESSION_LEVEL: fast # can also be set to fastest, fast, slow and slowest. If just enabling fastzip is not enough try setting this to fastest or fast.
CACHE_COMPRESSION_LEVEL: fastest # same as above, but for caches
TRANSFER_METER_FREQUENCY: 5s # will display transfer progress every 5 seconds for artifacts and remote caches.
# cache dependencies and build environment to speed up setup # cache dependencies and build environment to speed up setup
cache: cache:
@ -26,10 +33,6 @@ cache:
- cargo/ - cargo/
- target/ - target/
# install build dependencies
before_script:
- apt-get update -yqq
- apt-get install -o dir::cache::archives="$APT_CACHE_DIR" -yqq --no-install-recommends capnproto build-essential cmake clang libclang-dev jq
.lints: .lints:
stage: lint stage: lint
@ -41,7 +44,6 @@ before_script:
lint:clippy: lint:clippy:
extends: .lints extends: .lints
script: script:
- rustup component add clippy
- cargo clippy -V - cargo clippy -V
- echo -e "\e[0Ksection_start:`date +%s`:clippy_output\r\e[0Kcargo clippy output" - echo -e "\e[0Ksection_start:`date +%s`:clippy_output\r\e[0Kcargo clippy output"
- cargo clippy -- --no-deps - cargo clippy -- --no-deps
@ -51,15 +53,14 @@ lint:clippy:
lint:fmt: lint:fmt:
extends: .lints extends: .lints
script: script:
- rustup component add rustfmt
- cargo fmt --version - cargo fmt --version
- echo -e "\e[0Ksection_start:`date +%s`:rustfmt_output\r\e[0KChanges suggested by rustfmt" - echo -e "\e[0Ksection_start:`date +%s`:rustfmt_output\r\e[0KChanges suggested by rustfmt"
- cargo fmt --check -- -v - cargo fmt --check -- -v
- echo -e "\e[0Ksection_end:`date +%s`:rustfmt_output\r\e[0K" - echo -e "\e[0Ksection_end:`date +%s`:rustfmt_output\r\e[0K"
# Check if the code builds on rust stable # Check if the code builds on rust stable
stable:build: stable:check:
stage: build stage: check
only: only:
- main - main
- development - development
@ -70,9 +71,94 @@ stable:build:
- cargo check --verbose - cargo check --verbose
- echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K" - echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K"
# Check if the code builds on rust stable on armv7
stable:check:armhf:
stage: check
only:
- main
- development
- merge_requests
before_script:
- mkdir -p $CARGO_HOME
- cp cargo-cross-config $CARGO_HOME/config.toml
script:
- rustc +stable --version && cargo --version
- echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo check with target armv7-unknown-linux-gnueabihf"
- cargo check --verbose --target armv7-unknown-linux-gnueabihf
- echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K"
# Check if the code builds on rust stable on arm64
stable:check:arm64:
stage: check
only:
- main
- development
- merge_requests
before_script:
- mkdir -p $CARGO_HOME
- cp cargo-cross-config $CARGO_HOME/config.toml
script:
- rustc +stable --version && cargo --version
- echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo check with target aarch64-unknown-linux-gnu"
- cargo check --verbose --target aarch64-unknown-linux-gnu
- echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K"
# Check if the code builds on rust stable
stable:build:amd64:
stage: build
only:
- main
- development
- merge_requests
script:
- rustc +stable --version && cargo --version
- echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo build with target x86_64-unknown-linux-gnu"
- cargo build --release --target x86_64-unknown-linux-gnu
- echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K"
artifacts:
paths:
- target/x86_64-unknown-linux-gnu/release/bffhd
# Check if the code builds on rust stable on armv7
stable:build:armhf:
stage: build
only:
- main
- development
before_script:
- mkdir -p $CARGO_HOME
- cp cargo-cross-config $CARGO_HOME/config.toml
script:
- rustc +stable --version && cargo --version
- echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo build with target armv7-unknown-linux-gnueabihf"
- cargo build --release --target armv7-unknown-linux-gnueabihf
- echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K"
artifacts:
paths:
- target/armv7-unknown-linux-gnueabihf/release/bffhd
# Check if the code builds on rust stable on arm64
stable:build:arm64:
stage: build
only:
- main
- development
before_script:
- mkdir -p $CARGO_HOME
- cp cargo-cross-config $CARGO_HOME/config.toml
script:
- rustc +stable --version && cargo --version
- echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo build with target aarch64-unknown-linux-gnu"
- cargo build --release --target aarch64-unknown-linux-gnu
- echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K"
artifacts:
paths:
- target/aarch64-unknown-linux-gnu/release/bffhd
stable:test: stable:test:
stage: build stage: build
needs: ["stable:build"] needs: ["stable:check"]
only: only:
- main - main
- development - development
@ -80,14 +166,12 @@ stable:test:
script: script:
- echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo test --no-run" - echo -e "\e[0Ksection_start:`date +%s`:build_output\r\e[0KOutput of cargo test --no-run"
- cargo test --verbose --no-run --workspace - cargo test --verbose --no-run --workspace
- echo -e "\e[0Ksection_end:`date +%s`:build_output\r\e[0K"
- cargo install --root $CARGO_HOME cargo2junit
.tests: .tests:
stage: test stage: test
needs: ["stable:test"] needs: ["stable:test"]
script: script:
- cargo test --workspace $TEST_TARGET -- -Z unstable-options --format json --report-time | $CARGO_HOME/bin/cargo2junit > report.xml - cargo test --workspace $TEST_TARGET -- -Z unstable-options --format json --report-time | cargo2junit > report.xml
artifacts: artifacts:
when: always when: always
reports: reports:
@ -114,6 +198,23 @@ unit test 3:3:
TEST_TARGET: "--examples" TEST_TARGET: "--examples"
extends: .tests extends: .tests
upload_binaries:
stage: release
image: curlimages/curl:latest
before_script: []
cache: []
dependencies:
- stable:build:amd64
- stable:build:armhf
- stable:build:arm64
script:
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file target/aarch64-unknown-linux-gnu/release/bffhd "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${CI_COMMIT_TAG}/bffhd_${VERSION}_linux_arm64"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file target/x86_64-unknown-linux-gnu/release/bffhd "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${CI_COMMIT_TAG}/bffhd_${VERSION}_linux_amd64"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file target/armv7-unknown-linux-gnueabihf/release/bffhd "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${VERSION}/bffhd_${VERSION}_linux_arm"'
rules:
- if: $CI_COMMIT_TAG =~ "release/.*"
when: never
- if: $CI_COMMIT_BRANCH == "main"
release_prepare: release_prepare:
stage: release stage: release
@ -144,32 +245,106 @@ release_job:
name: "BFFH $VERSION" name: "BFFH $VERSION"
description: "GitLab CI auto-created release" description: "GitLab CI auto-created release"
tag_name: "release/$VERSION" tag_name: "release/$VERSION"
assets:
links:
- name: 'bffhd AMD64'
url: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${VERSION}/bffhd_${VERSION}_linux_amd64"
- name: 'bffhd ARMv7'
url: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${VERSION}/bffhd_${VERSION}_linux_arm"
- name: 'bffhd ARM64'
url: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/bffhd/${VERSION}/bffhd_${VERSION}_linux_arm64"
build:docker-releases: build:docker-releases:
stage: dockerify stage: dockerify
image: image: jdrouet/docker-with-buildx:latest
name: gcr.io/kaniko-project/executor:v1.6.0-debug dependencies:
entrypoint: [""] - stable:build:amd64
- stable:build:armhf
- stable:build:arm64
tags:
- linux
- docker
- fabinfra
variables:
DOCKER_HOST: tcp://docker:2375/
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
TRIVY_NO_PROGRESS: "true"
TRIVY_CACHE_DIR: ".trivycache/"
services:
- docker:dind
before_script: before_script:
- '' - export TRIVY_VERSION=$(wget -qO - "https://api.github.com/repos/aquasecurity/trivy/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/')
- echo $TRIVY_VERSION
- wget --no-verbose https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz -O - | tar -zxvf -
script: script:
- mkdir -p /kaniko/.docker - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
- /kaniko/executor --force --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG - docker buildx create --name cibuilder --driver docker-container --use
- docker buildx ls
- docker buildx inspect --bootstrap
- docker buildx build --platform linux/arm/v7,linux/arm64,linux/amd64 -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG .
- docker buildx build --load --platform linux/amd64 -t $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG .
# Build report
- ./trivy image --exit-code 0 --format template --template "@contrib/gitlab.tpl" -o gl-container-scanning-report.json $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
# Print report
- ./trivy image --exit-code 0 --severity HIGH $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
# Fail on severe vulnerabilities
- ./trivy image --exit-code 1 --severity CRITICAL $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
- docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
cache:
paths:
- .trivycache/
artifacts:
reports:
container_scanning: gl-container-scanning-report.json
rules: rules:
- if: $CI_COMMIT_TAG =~ "release/.*" - if: $CI_COMMIT_TAG =~ "release/.*"
when: never when: never
build:docker-development: build:docker-development:
stage: dockerify stage: dockerify
image: image: jdrouet/docker-with-buildx:latest
name: gcr.io/kaniko-project/executor:v1.6.0-debug dependencies:
entrypoint: [""] - stable:build:amd64
- stable:build:armhf
- stable:build:arm64
tags:
- linux
- docker
- fabinfra
variables:
DOCKER_HOST: tcp://docker:2375/
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
TRIVY_NO_PROGRESS: "true"
TRIVY_CACHE_DIR: ".trivycache/"
services:
- docker:dind
before_script: before_script:
- '' - export TRIVY_VERSION=$(wget -qO - "https://api.github.com/repos/aquasecurity/trivy/releases/latest" | grep '"tag_name":' | sed -E 's/.*"v([^"]+)".*/\1/')
- echo $TRIVY_VERSION
- wget --no-verbose https://github.com/aquasecurity/trivy/releases/download/v${TRIVY_VERSION}/trivy_${TRIVY_VERSION}_Linux-64bit.tar.gz -O - | tar -zxvf -
script: script:
- mkdir -p /kaniko/.docker - docker login $CI_REGISTRY -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD"
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
- /kaniko/executor --force --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:dev-latest - docker buildx create --name cibuilder --driver docker-container --use
- docker buildx ls
- docker buildx inspect --bootstrap
- docker buildx build --platform linux/arm/v7,linux/arm64,linux/amd64 -t $CI_REGISTRY_IMAGE:development .
- docker buildx build --load --platform linux/amd64 -t $CI_REGISTRY_IMAGE:development .
# Build report
- ./trivy image --exit-code 0 --format template --template "@contrib/gitlab.tpl" -o gl-container-scanning-report.json $CI_REGISTRY_IMAGE:development
# Print report
- ./trivy image --exit-code 0 --severity HIGH $CI_REGISTRY_IMAGE:development
# Fail on severe vulnerabilities
- ./trivy image --exit-code 1 --severity CRITICAL $CI_REGISTRY_IMAGE:development
- docker push $CI_REGISTRY_IMAGE:development
cache:
paths:
- .trivycache/
artifacts:
reports:
container_scanning: gl-container-scanning-report.json
only: only:
- development - development

29
Dockerfile
View File

@ -1,22 +1,23 @@
# Setup build image for multistage build FROM --platform=$BUILDPLATFORM alpine:latest as copy
FROM rust:bullseye as builder ARG TARGETPLATFORM
# install build deps RUN case "$TARGETPLATFORM" in \
RUN apt-get update && apt-get upgrade -y "linux/arm/v7") echo armv7-unknown-linux-gnueabihf > /rust_target.txt ;; \
RUN apt-get install -yqq --no-install-recommends capnproto python3 python3-pip "linux/arm/v6") echo arm-unknown-linux-gnueabihf > /rust_target.txt ;; \
RUN pip3 install paho-mqtt "linux/arm64") echo aarch64-unknown-linux-gnu > /rust_target.txt ;; \
"linux/amd64") echo x86_64-unknown-linux-gnu > /rust_target.txt ;; \
*) exit 1 ;; \
esac
WORKDIR /usr/src/bffh WORKDIR /usr/src/bffh
COPY . . COPY . .
RUN cargo build --release RUN cp target/$(cat /rust_target.txt)/release/bffhd ./bffhd.bin
# Setup deployable image # Setup deployable image
FROM debian:bullseye-slim FROM ubuntu:22.04
# Install runtime deps RUN apt-get update && apt-get upgrade -y
#RUN apt-get update && apt-get upgrade -yqq RUN apt-get install -yqq --no-install-recommends python3 python3-pip
COPY --from=builder /usr/src/bffh/target/release/bffhd /usr/local/bin/bffhd RUN pip3 install paho-mqtt
#COPY --from=builder /usr/src/bffh/examples/bffh.dhall /etc/diflouroborane.dhall COPY --from=copy /usr/src/bffh/bffhd.bin /usr/local/bin/bffhd
# RUN diflouroborane --print-default > /etc/diflouroborane.toml
VOLUME /etc/bffh/ VOLUME /etc/bffh/
VOLUME /var/lib/bffh/ VOLUME /var/lib/bffh/
VOLUME /usr/local/lib/bffh/adapters/ VOLUME /usr/local/lib/bffh/adapters/

4
bffhd/resources/state/value.rs
View File

@ -275,10 +275,6 @@ pub struct ImplDebugInfo {
/// [statevalue_register](macro@crate::statevalue_register) macro with your OID as first and type /// [statevalue_register](macro@crate::statevalue_register) macro with your OID as first and type
/// as second parameter like so: /// as second parameter like so:
/// ///
/// ```no_run
/// struct MyStruct;
/// statevalue_register!(ObjectIdentifier::from_str("1.3.6.1.4.1.48398.612.1.14").unwrap(), MyStruct)
/// ```
pub struct ImplEntry<'a> { pub struct ImplEntry<'a> {
id: ImplId<'a>, id: ImplId<'a>,
data: ImplData<'a>, data: ImplData<'a>,

8
cargo-cross-config Normal file
View File

@ -0,0 +1,8 @@
[target.armv7-unknown-linux-gnueabihf]
linker = "arm-linux-gnueabihf-gcc"
[target.arm-unknown-linux-gnueabihf]
linker = "arm-linux-gnueabi-gcc"
[target.aarch64-unknown-linux-gnu]
linker = "aarch64-linux-gnu-gcc"

3
runtime/lightproc/examples/proc_panic.rs
View File

@ -30,7 +30,8 @@ where
} }
let schedule = |t| (QUEUE.deref()).send(t).unwrap(); let schedule = |t| (QUEUE.deref()).send(t).unwrap();
let (proc, handle) = LightProc::recoverable(future, schedule); let span = tracing::trace_span!("runtime.spawn", kind = "local");
let (proc, handle) = LightProc::recoverable(future, schedule, span, None);
let handle = handle.on_panic( let handle = handle.on_panic(
|err: Box<dyn Any + Send>| match err.downcast::<&'static str>() { |err: Box<dyn Any + Send>| match err.downcast::<&'static str>() {

3
runtime/lightproc/examples/proc_run.rs
View File

@ -17,7 +17,8 @@ where
let future = async move { fut.await }; let future = async move { fut.await };
let schedule = move |t| sender.send(t).unwrap(); let schedule = move |t| sender.send(t).unwrap();
let (proc, handle) = LightProc::build(future, schedule); let span = tracing::trace_span!("runtime.spawn", kind = "local");
let (proc, handle) = LightProc::build(future, schedule, span, None);
proc.schedule(); proc.schedule();

17
runtime/lightproc/src/lightproc.rs
View File

@ -9,6 +9,7 @@
//! # Example Usage //! # Example Usage
//! //!
//! ```rust //! ```rust
//! use tracing::Span;
//! use lightproc::prelude::*; //! use lightproc::prelude::*;
//! //!
//! // ... future that does work //! // ... future that does work
@ -23,6 +24,8 @@
//! let panic_recoverable = LightProc::recoverable( //! let panic_recoverable = LightProc::recoverable(
//! future, //! future,
//! schedule_function, //! schedule_function,
//! Span::current(),
//! None,
//! ); //! );
//! ``` //! ```
@ -60,6 +63,7 @@ impl LightProc {
/// # Example /// # Example
/// ```rust /// ```rust
/// # use std::any::Any; /// # use std::any::Any;
/// # use tracing::Span;
/// # use lightproc::prelude::*; /// # use lightproc::prelude::*;
/// # /// #
/// # // ... basic schedule function with no waker logic /// # // ... basic schedule function with no waker logic
@ -72,9 +76,11 @@ impl LightProc {
/// let (proc, handle) = LightProc::recoverable( /// let (proc, handle) = LightProc::recoverable(
/// future, /// future,
/// schedule_function, /// schedule_function,
/// Span::current(),
/// None
/// ); /// );
/// let handle = handle.on_panic(|s: &mut EmptyProcState, e: Box<dyn Any + Send>| { /// let handle = handle.on_panic(|e: Box<dyn Any + Send>| {
/// let reason = e.downcast::<String>(); /// let reason = e.downcast::<String>().unwrap();
/// println!("future panicked!: {}", &reason); /// println!("future panicked!: {}", &reason);
/// }); /// });
/// ``` /// ```
@ -110,13 +116,6 @@ impl LightProc {
/// # // ... basic schedule function with no waker logic /// # // ... basic schedule function with no waker logic
/// # fn schedule_function(proc: LightProc) {;} /// # fn schedule_function(proc: LightProc) {;}
/// # /// #
/// # // ... process stack with a lifecycle callback
/// # let proc_stack =
/// # ProcStack::default()
/// # .with_after_panic(|s: &mut EmptyProcState| {
/// # println!("After panic started!");
/// # });
/// #
/// // ... creating a standard process /// // ... creating a standard process
/// let standard = LightProc::build( /// let standard = LightProc::build(
/// future, /// future,

14
runtime/lightproc/src/recoverable_handle.rs
View File

@ -49,8 +49,7 @@ impl<R> RecoverableHandle<R> {
/// ///
/// ```rust /// ```rust
/// # use std::any::Any; /// # use std::any::Any;
/// use lightproc::proc_stack::ProcStack; /// # use tracing::Span;
/// use lightproc::proc_state::EmptyProcState;
/// # use lightproc::prelude::*; /// # use lightproc::prelude::*;
/// # /// #
/// # // ... future that does work /// # // ... future that does work
@ -61,21 +60,16 @@ impl<R> RecoverableHandle<R> {
/// # // ... basic schedule function with no waker logic /// # // ... basic schedule function with no waker logic
/// # fn schedule_function(proc: LightProc) {;} /// # fn schedule_function(proc: LightProc) {;}
/// # /// #
/// # // ... process stack with a lifecycle callback
/// # let proc_stack =
/// # ProcStack::default()
/// # .with_after_panic(|s: &mut EmptyProcState| {
/// # println!("After panic started!");
/// # });
/// #
/// // ... creating a recoverable process /// // ... creating a recoverable process
/// let (proc, recoverable) = LightProc::recoverable( /// let (proc, recoverable) = LightProc::recoverable(
/// future, /// future,
/// schedule_function, /// schedule_function,
/// Span::current(),
/// None
/// ); /// );
/// ///
/// recoverable /// recoverable
/// .on_return(|_e: Box<dyn Any + Send>| { /// .on_panic(|_e: Box<dyn Any + Send>| {
/// println!("Inner future panicked"); /// println!("Inner future panicked");
/// }); /// });
/// ``` /// ```