From e210caf77edf659ae9ddf3eaed0d312e66c4567d Mon Sep 17 00:00:00 2001
From: Nadja Reitzenstein <me@dequbed.space>
Date: Fri, 5 May 2023 13:41:31 +0200
Subject: [PATCH] add initial capnp console-api schema

---
 runtime/console-api/Cargo.toml              |   8 +
 runtime/console-api/schema/async_ops.capnp  |  59 +++++
 runtime/console-api/schema/common.capnp     | 228 ++++++++++++++++++++
 runtime/console-api/schema/instrument.capnp |  50 +++++
 runtime/console-api/schema/resources.capnp  |  98 +++++++++
 runtime/console-api/schema/stream.capnp     |  12 ++
 runtime/console-api/schema/tasks.capnp      | 160 ++++++++++++++
 runtime/console-api/schema/trace.capnp      |  80 +++++++
 runtime/console-api/src/lib.rs              |  14 ++
 9 files changed, 709 insertions(+)
 create mode 100644 runtime/console-api/Cargo.toml
 create mode 100644 runtime/console-api/schema/async_ops.capnp
 create mode 100644 runtime/console-api/schema/common.capnp
 create mode 100644 runtime/console-api/schema/instrument.capnp
 create mode 100644 runtime/console-api/schema/resources.capnp
 create mode 100644 runtime/console-api/schema/stream.capnp
 create mode 100644 runtime/console-api/schema/tasks.capnp
 create mode 100644 runtime/console-api/schema/trace.capnp
 create mode 100644 runtime/console-api/src/lib.rs

diff --git a/runtime/console-api/Cargo.toml b/runtime/console-api/Cargo.toml
new file mode 100644
index 0000000..d142858
--- /dev/null
+++ b/runtime/console-api/Cargo.toml
@@ -0,0 +1,8 @@
+[package]
+name = "console-api"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
diff --git a/runtime/console-api/schema/async_ops.capnp b/runtime/console-api/schema/async_ops.capnp
new file mode 100644
index 0000000..550a2ca
--- /dev/null
+++ b/runtime/console-api/schema/async_ops.capnp
@@ -0,0 +1,59 @@
+@0xda4fc7f2ce698a66;
+
+using Common = import "common.capnp";
+
+struct Update {
+    newAsyncOps @0 :List(AsyncOp);
+    # List of new async ops that were crated since the last Update.
+
+    statsUpdate @1 :Common.IntMap(Stats);
+    # All async op stats that have changed since the last Update was sent.
+
+    droppedEvents @2 :UInt64;
+    # Count of how many async op events were not recorded because the applications event buffer was
+    # at capacity.
+    #
+    # If everything is working normally, this is 0. If it is not that may inidcate that some data is
+    # missing from this update and it may be necessary to increase the number of events buffered by
+    # the application to ensure that data loss is avoided.
+    #
+    # If the application's instrumentation ensures reliable delivery of events, this will alway be 0
+}
+
+struct AsyncOp {
+    id @0 :Common.Id;
+    # The async op's ID.
+    #
+    # This uniquely identifies this op across all *currently live* ones.
+
+    metadata @1 :Common.MetaId;
+    # The numeric id of the op's Metadata
+    #
+    # This identifies the `Metadata` that describes the `tracing` span
+    # corresponding to this async op. The metadata for this ID will have been sent
+    # in a prior `RegisterMetadata` message.
+
+    source @2 :Text;
+    # The source of this async operation. Most commonly this should be the name of the method where
+    # the instantiation of this op has happened.
+
+    parentAsyncOp @3 :Common.Id;
+    # The ID of the parent async op.
+    #
+    # This field is only set if this async op was created while inside of another async op.
+
+    resourceId @4 :Common.Id;
+    # The resource's ID
+}
+
+struct Stats {
+    createdAt @0 :Common.Timestamp;
+    droppedAt @1 :Common.Timestamp;
+
+    task @2 :Common.Id;
+    # The ID of the task currently awaiting on this op
+
+    pollStats @3 :Common.PollStats;
+
+    attributes @4 :List(Common.Attribute);
+}
diff --git a/runtime/console-api/schema/common.capnp b/runtime/console-api/schema/common.capnp
new file mode 100644
index 0000000..60d9d7a
--- /dev/null
+++ b/runtime/console-api/schema/common.capnp
@@ -0,0 +1,228 @@
+@0x911648bc10d7e94f;
+
+struct Timestamp {
+    # Timestamp in consisting of a 64-bit second counter and a 32-bit nanosecond counter
+
+    seconds @0 :Int64;
+    # Non-leap second counter since 1970-01-01T00:00:00Z
+
+    nanoseconds @1 :UInt32;
+    # Nanosecond counter since the last non-leap second
+}
+
+struct Duration {
+    # Duration struct consiting of a 64-bit seconds value and a 32-bit *signed* nanosecond value
+
+    seconds @0 :Int64;
+    # Seconds of duration
+
+    nanoseconds @1 :Int32;
+    # Nanoseconds of duration
+}
+
+struct Optional(T) {
+    union {
+        nothing @0 :Void;
+        just @1 :T;
+    }
+}
+
+struct Id {
+    id @0 :UInt64;
+}
+
+struct MetaId {
+    id @0 :UInt64;
+}
+
+struct SpanId {
+    id @0 :UInt64;
+}
+
+struct Field {
+    name :union {
+        string @0 :Text;
+        # Name as immediate string
+        index @1 :UInt64;
+        # An index position into the `Metadata.field_names` of the metadata for
+        # the task span that the field came from
+    }
+
+    value :union {
+        debug @2 :Text;
+        # Value serialized to string using `fmt::Debug`
+        string @3 :Text;
+        # A string value
+        u64 @4 :UInt64;
+        # An unsigned integer value
+        i64 @5 :Int64;
+        # A signed integer value
+        bool @6 :Bool;
+        # A boolean value
+    }
+
+    metadataId @7 :MetaId;
+    # Metadata for the task span that the field came from
+}
+
+struct Span {
+    # Represents a period of time in which a program was executing in a particular context.
+    #
+    # Corresponds to `Span` in the `tracing` crate.
+
+    id @0 :SpanId;
+    # An id that unique identifies this span in relation to other spans
+
+    metadataId @1 :MetaId;
+    # Identifier for metadata describing static characteristics of all spans originating
+    # from that callsite, such as its name, source code location, verbosity level, and
+    # the names of its fields.
+
+    fields @2 :List(Field);
+    # User-defined key-value pairs of arbitrary data that describe the context the span represents
+
+    at @3 :Timestamp;
+    # Timestamp for the span
+}
+
+struct Location {
+    file @0 :Text;
+    # File path
+    modulePath @1 :Text;
+    # Rust module path
+    line @2 :UInt32;
+    # Line number
+    column @3 :UInt32;
+}
+
+struct RegisterMetadata {
+    # Any new metadata that was registered since the last update
+
+    metadata @0 :List(NewMetadata);
+    # The new metadata that was registered
+
+    struct NewMetadata {
+        id @0 :MetaId;
+        # Unique identifier for the metadata
+
+        metadata @1 :Metadata;
+        # Metadata payload
+    }
+}
+
+struct Metadata {
+    # Metadata associated with a span or an event
+
+    name @0 :Text;
+    # Name of the span or event
+
+    target @1 :Text;
+    # Describes the part of the system where the span or event that this metadata describes occured
+
+    modulePath @2 :Text;
+    # Path to the Rust module where the span occured
+
+    location @3 :Location;
+    # Rust source code location associated with this span or event
+
+    kind @4 :Kind;
+    # Indicates wheter this metadata is associated with a span or an event
+    enum Kind {
+        span @0;
+        event @1;
+    }
+
+    level @5 :Level;
+    # Describes the level of verbosity of a span or event
+    enum Level {
+        error @0;
+        # Designates very serious errors
+
+        warn @1;
+        # Designates hazardous situations
+
+        info @2;
+        # Designates useful information
+
+        debug @3;
+        # Designates lower priority information
+
+        trace @4;
+        # Designates very low priority, often extremely verbose, information
+    }
+}
+
+struct PollStats {
+    # Contains stats about objects that can be polled. Currently these can be:
+    # - tasks that have been spawned
+    # - async operations that are performed within the context of a task
+
+    polls @0 :UInt64;
+    # The total number of times this object was polled
+
+    firstPoll @1 :Optional(Timestamp);
+    # The timestamp of the first time this object was polled.
+    #
+    # If this is `Nothing`, the object has not yet been polled.
+    #
+    # Subtracting this timestamp from `created_at` can be used to calculate the
+    # time to first poll for this object, a measurement of executor latency.
+
+    lastPollStarted @2 :Optional(Timestamp);
+    # The timestamp of the most recent time this objects's poll method was invoked.
+    #
+    # If this is `Nothing`, the object has not yet been polled.
+    #
+    # If the object has only been polled a single time, then this value may be
+    # equal to the `first_poll` timestamp.
+
+    lastPollEnded @3 :Optional(Timestamp);
+    # The timestamp of the most recent time this objects's poll method finished execution.
+    #
+    # If this is `Nothing`, the object has not yet been polled or is currently being polled.
+    #
+    # If the object does not exist anymore, then this is the time the final invocation of
+    # its poll method has completed.
+
+    busyTime @4 :Duration;
+    # The total duration this object was being *actively polled*, summed across
+    # all polls.
+    #
+    # Note that this includes only polls that have completed, and does not
+    # reflect any in-progress polls. Subtracting `busy_time` from the
+    # total lifetime of the polled object results in the amount of time it
+    # has spent *waiting* to be polled (including the `scheduled_time` value
+    # from `TaskStats`, if this is a task).
+}
+
+struct Attribute {
+    # State attributes of an entity. These are dependent on the type of the entity.
+    #
+    # For example, a timer resource will have a duration, while a semaphore resource may
+    # have a permit count. Likewise, the async ops of a semaphore may have attributes
+    # indicating how many permits they are trying to acquire vs how many are acquired.
+    # These values may change over time. Therefore, they live in the runtime stats rather
+    # than the static data describing the entity.
+
+    field @0 :Field;
+    # The key-value pair for the attribute
+
+    unit @1 :Text;
+    # Some values carry an unit of measurement, e.g. durations given in 'ms'.
+}
+
+struct Map(K, V) {
+    entries @0 :List(Entry);
+    struct Entry {
+        key @0 :K;
+        value @1 :V;
+    }
+}
+
+struct IntMap(V) {
+    entries @0 :List(Entry);
+    struct Entry {
+        key @0 :UInt64;
+        value @1 :V;
+    }
+}
diff --git a/runtime/console-api/schema/instrument.capnp b/runtime/console-api/schema/instrument.capnp
new file mode 100644
index 0000000..6a004fe
--- /dev/null
+++ b/runtime/console-api/schema/instrument.capnp
@@ -0,0 +1,50 @@
+@0xd53fadcd6c8f437f;
+
+using Common = import "common.capnp";
+using Stream = import "stream.capnp";
+using Tasks = import "tasks.capnp";
+using Resources = import "resources.capnp";
+using AsyncOps = import "async_ops.capnp";
+
+interface Instrument {
+    watchUpdates @0 (request :InstrumentRequest, receiver :Stream.Receiver(Update)) -> (stream :Stream.Sender);
+    watchTaskDetails @1 (request :TaskDetailsRequest, receiver :Stream.Receiver(Tasks.TaskDetails)) -> (stream :Stream.Sender);
+    pause @2 PauseRequest -> PauseResponse;
+    resume @3 ResumeRequest -> ResumeResponse;
+}
+
+struct InstrumentRequest {
+
+}
+
+struct Update {
+    now @0 :Common.Timestamp;
+
+    taskUpdate @1 :Tasks.Update;
+
+    resourceUpdate @2 :Resources.Update;
+
+    asyncOpUpdate @3 :AsyncOps.Update;
+
+    newMetadata @4 :Common.RegisterMetadata;
+}
+
+struct TaskDetailsRequest {
+    id @0 :Common.Id;
+}
+
+struct PauseRequest {
+
+}
+
+struct PauseResponse {
+
+}
+
+struct ResumeRequest {
+
+}
+
+struct ResumeResponse {
+
+}
diff --git a/runtime/console-api/schema/resources.capnp b/runtime/console-api/schema/resources.capnp
new file mode 100644
index 0000000..82e7f9f
--- /dev/null
+++ b/runtime/console-api/schema/resources.capnp
@@ -0,0 +1,98 @@
+@0xd75a34c00d06ab36;
+
+using Common = import "common.capnp";
+
+struct Update {
+    newResource @0 :List(Resource);
+    # List of new resources that were created since the last Update was sent
+
+    statsUpdate @1 :Common.IntMap(Stats);
+    # All resource stats that have changed since the last update
+
+    newPollOps @2 :List(PollOp);
+    # List of all new poll ops that have been invoked on resources since the last update
+
+    droppedEvents @3 :UInt64;
+    # Count of how many resource events were not recorded because the applications event buffer was
+    # at capacity.
+    #
+    # If everything is working normally, this is 0. If it is not that may inidcate that some data is
+    # missing from this update and it may be necessary to increase the number of events buffered by
+    # the application to ensure that data loss is avoided.
+    #
+    # If the application's instrumentation ensures reliable delivery of events, this will alway be 0
+}
+
+struct Resource {
+    id @0 :Common.Id;
+    # The resource's ID.
+    #
+    # This uniquely identifies this resource across all *currently live* resoures.
+
+    metadata @1 :Common.MetaId;
+    # Numeric ID of the resource's Metadata.
+
+    concreteType @2 :Text;
+    # Name of the concrete Rust type of this resource
+
+    kind @3 :Kind;
+    # The kind of this resource
+    struct Kind {
+        union {
+            wellKnown @0 :WellKnown;
+            other @1 :Text;
+        }
+
+        enum WellKnown {
+            timer @0;
+        }
+    }
+
+    location @4 :Common.Location;
+    # The location in code where this resource was created
+
+    parentResourceId @5 :Common.Id;
+    # ID of the parent resource
+
+    internal @6 :Bool;
+    # Wether or not this resource is an internal component of another resource
+}
+
+struct Stats {
+    # Task runtime stats of a resource
+
+    createdAt @0 :Common.Timestamp;
+    # Timestamp of when the resource was created
+
+    droppedAt @1 :Common.Timestamp;
+    # Timestamp of when the resource was dropped
+
+    attributes @2 :List(Common.Attribute);
+    # State attributes of the resource. These are dependent on the type of the resource. For example
+    # a timer resource will have a duration, while a semaphore resource may have permits as an
+    # attribute. These values may change over time as the state of the resource changes. Therefore
+    # they live in the runtime stats rather than the static data describing the resource
+}
+
+struct PollOp {
+    metadata @1 :Common.MetaId;
+    # Numeric ID of the op's Metadata
+    # 
+    # This identifies the `Metadata` that describes the `tracing` span corresponding to this op. The
+    # metadata for this ID will have been sent in a prior `RegisterMetadata` message.
+
+    resource @0 :Common.Id;
+    # The resource's ID
+
+    name @2 :Text;
+    # Name of this op (e.g. poll_elapsed, new_timeout, reset, etc.)
+
+    task @3 :Common.Id;
+    # The ID of the task context that this poll op has been called from
+
+    asyncOp @4 :Common.Id;
+    # The ID of the async op that this poll op is part of
+
+    ready @5 :Bool;
+    # Wheter this poll op has return with `Ready` or not.
+}
diff --git a/runtime/console-api/schema/stream.capnp b/runtime/console-api/schema/stream.capnp
new file mode 100644
index 0000000..68f5dd3
--- /dev/null
+++ b/runtime/console-api/schema/stream.capnp
@@ -0,0 +1,12 @@
+@0x85894bd060182ed3;
+
+interface Receiver(T) {
+    send @0 (chunk :T) -> stream;
+    done @1 ();
+}
+
+interface Sender {
+    pause @0 ();
+    resume @1 ();
+    stop @2 ();
+}
diff --git a/runtime/console-api/schema/tasks.capnp b/runtime/console-api/schema/tasks.capnp
new file mode 100644
index 0000000..863aa11
--- /dev/null
+++ b/runtime/console-api/schema/tasks.capnp
@@ -0,0 +1,160 @@
+@0xef8effe12d55773f;
+
+using Common = import "common.capnp";
+
+struct Task {
+    # Data recorded when a new task is spawned
+
+    id @0 :Common.Id;
+    # The task's ID.
+    #
+    # This uniquely identifies this task across all *currently live* tasks.
+    # When the task's stats change, or when the task completes, it will be
+    # identified by this ID; if the client requires additional information
+    # included in the `Task` message, it should store that data and access it
+    # by ID.
+
+    metadata @1 :Common.MetaId;
+    # The numeric ID of the task's `Metadata`.
+    #
+    # This identifies the `Metadata` that describes the `tracing` span
+    # corresponding to this task. The metadata for this ID will have been sent
+    # in a prior `RegisterMetadata` message.
+
+    kind @2 :Kind;
+    # The category of task this task belongs to.
+    enum Kind {
+        spawn @0;
+        # Spawned using the main runtime asyncronous task spawning
+        blocking @1;
+        # Spawning as a blocking operation
+    }
+
+    fields @3 :List(Common.Field);
+    # A list of 'fields' attached to this task.
+
+    parents @4 :List(Common.SpanId);
+    # An ordered list of span IDs corresponding to the `tracing` span context
+    # in which this task was spawned.
+    #
+    # The first span ID in this list is the immediate parent, followed by that
+    # span's parent, and so on. The final ID is the root span of the current
+    # trace.
+    #
+    # If this is empty, there were *no* active spans when the task was spawned.
+    #
+    # These IDs may correspond to `tracing` spans which are *not* tasks, if
+    # additional trace data is being collected.
+
+    location @5 :Common.Location;
+    # The location in the code where this task was spawned.
+}
+
+struct Update {
+    # A task state update.
+    #
+    # Each `Update` contains any task data that has changed since the last
+    # update. This includes:
+    # - any new tasks that were spawned since the last update
+    # - the current stats for any task whose stats changed since the last update
+
+    newTasks @0 :List(Task);
+    # A list of new tasks that were spawned since the last `Update` was
+    # sent.
+    #
+    # If this is empty, no new tasks were spawned.
+
+    statsUpdate @1 :Common.IntMap(Stats);
+    # Any task stats that have changed since the last update.
+    #
+    # This is a map of task IDs (64-bit unsigned integers) to task stats. If a
+    # task's ID is not included in this map, then its stats have *not* changed
+    # since the last `Update` in which they were present. If a task's ID
+    # *is* included in this map, the corresponding value represents a complete
+    # snapshot of that task's stats at in the current time window.
+
+    droppedEvents @2 :UInt64;
+    # A count of how many task events (e.g. polls, spawns, etc) were not
+    # recorded because the application's event buffer was at capacity.
+    #
+    # If everything is working normally, this should be 0. If it is greater
+    # than 0, that may indicate that some data is missing from this update, and
+    # it may be necessary to increase the number of events buffered by the
+    # application to ensure that data loss is avoided.
+    #
+    # If the application's instrumentation ensures reliable delivery of events,
+    # this will always be 0.
+}
+
+struct TaskDetails {
+    # A task details update
+
+    id @0 :Common.Id;
+    # The tasks id to which the details belong to
+
+    now @1 :Common.Timestamp;
+    # The timestamp for when the update to the task took place
+
+    pollTimesHistogram @2 :DurationHistogram;
+    # A histogram of task poll durations
+
+    scheduledTimesHistogram @3 :DurationHistogram;
+    # A histogram of task scheduled durations
+    #
+    # The scheduled duration is the time a task spends between woken and when it is next polled
+}
+
+struct Stats {
+    # Task performance statistics
+
+    createdAt @0 :Common.Timestamp;
+    # Timestamp of when the task was created
+
+    droppedAt @1 :Common.Timestamp;
+    # Timestamp of when the task was dropped
+
+    wakes @2 :UInt64;
+    # Total number of times this task has been woken over its lifetime
+
+    wakerClones @3 :UInt64;
+    # Total number of times this tasks waker was cloned
+
+    wakerDrops @4 :UInt64;
+    # Total number of times this tasks waker was dropped
+
+    lastWake @5 :Common.Optional(Common.Timestamp);
+    # The timestamp of the most recent time this task has been woken.
+    #
+    # If this is `Nothing`, the task has not yet been woken
+
+    pollStats @6 :Common.PollStats;
+    # Contains task poll statistics
+
+    selfWakes @7 :UInt64;
+    # Total number of times this task has woken itself
+
+    scheduledTime @8 :Common.Duration;
+    # The total duration this task was scheduled prior to being polled, summed
+    # across all poll cycles.
+    #
+    # Note that this includes only polls that have started, and does not
+    # reflect any scheduled state where the task hasn't yet been polled.
+    # Subtracting both `busy_time` (from the task's `PollStats`) and
+    # `scheduled_time` from the total lifetime of the task results in the
+    # amount of time it spent unable to progress because it was waiting on
+    # some resource.
+}
+
+struct DurationHistogram {
+    rawHistogram @0 :Data;
+    # HdrHistogram.rs `Histogram` serialized to binary in the V2 format
+
+    maxValue @1 :UInt64;
+    # The histograms maximum value
+
+    highOutliers @2 :UInt64;
+    # The number of outliers which have exceeded the histograms maximum value
+
+    highestOutlier @3 :UInt64;
+    # The highest recorded outlier. This only has a sensible value if `highOutliers` is greater than zero
+}
diff --git a/runtime/console-api/schema/trace.capnp b/runtime/console-api/schema/trace.capnp
new file mode 100644
index 0000000..aaf1a7a
--- /dev/null
+++ b/runtime/console-api/schema/trace.capnp
@@ -0,0 +1,80 @@
+@0xb021b19fd5986342;
+
+using Common = import "common.capnp";
+using Stream = import "stream.capnp";
+
+interface Trace {
+    watch @0 (request :WatchRequest, receiver :Stream.Receiver(TraceEvent)) -> (stream :Stream.Sender);
+}
+
+struct WatchRequest {
+    # Start watching trace events with the provided filter
+
+    filter @0 :Text;
+    # Filters which trace events should be streamed
+}
+
+struct TraceEvent {
+    union {
+        registerThread @0 :RegisterThreads;
+        # A new thread was registered
+
+        registerMetadata @1 :Common.RegisterMetadata;
+        # A new span metadata was registered
+
+        newSpan @2 :Common.Span;
+        # A span was created
+
+        enterSpan @3 :Enter;
+        # A span was entered
+
+        exitSpan @4 :Exit;
+        # A span was exited
+
+        closeSpan @5 :Close;
+        # A span was closed
+    }
+
+    struct RegisterThreads {
+        # Signals that a new thread was registered
+
+        names @0 :Common.IntMap(Text);
+        # Maps the registered thread id's to their associated name
+    }
+
+    struct Enter {
+        # Signals that a span was entered
+
+        spanId @0 :Common.SpanId;
+        # Identified the span that was entered
+
+        threadId @1 :UInt64;
+        # Identifies who entered the span
+
+        at @2 :Common.Timestamp;
+        # Identifies when the span was entered
+    }
+
+    struct Exit {
+        # Signals that a span was exited
+        
+        spanId @0 :Common.SpanId;
+        # Identifies the spawn that was exited
+
+        threadId @1 :UInt64;
+        # Identifies who exited the span
+
+        at @2 :Common.Timestamp;
+        # Identifies when the span was exited
+    }
+
+    struct Close {
+        # Signals that a span was closed
+
+        spanId @0 :Common.SpanId;
+        # Identifies the span that was closed
+
+        at @1 :Common.Timestamp;
+        # Identifies when the span was closed
+    }
+}
diff --git a/runtime/console-api/src/lib.rs b/runtime/console-api/src/lib.rs
new file mode 100644
index 0000000..7d12d9a
--- /dev/null
+++ b/runtime/console-api/src/lib.rs
@@ -0,0 +1,14 @@
+pub fn add(left: usize, right: usize) -> usize {
+    left + right
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn it_works() {
+        let result = add(2, 2);
+        assert_eq!(result, 4);
+    }
+}