diff --git a/runtime/console-api/Cargo.toml b/runtime/console-api/Cargo.toml new file mode 100644 index 0000000..d142858 --- /dev/null +++ b/runtime/console-api/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "console-api" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/runtime/console-api/schema/async_ops.capnp b/runtime/console-api/schema/async_ops.capnp new file mode 100644 index 0000000..550a2ca --- /dev/null +++ b/runtime/console-api/schema/async_ops.capnp @@ -0,0 +1,59 @@ +@0xda4fc7f2ce698a66; + +using Common = import "common.capnp"; + +struct Update { + newAsyncOps @0 :List(AsyncOp); + # List of new async ops that were crated since the last Update. + + statsUpdate @1 :Common.IntMap(Stats); + # All async op stats that have changed since the last Update was sent. + + droppedEvents @2 :UInt64; + # Count of how many async op events were not recorded because the applications event buffer was + # at capacity. + # + # If everything is working normally, this is 0. If it is not that may inidcate that some data is + # missing from this update and it may be necessary to increase the number of events buffered by + # the application to ensure that data loss is avoided. + # + # If the application's instrumentation ensures reliable delivery of events, this will alway be 0 +} + +struct AsyncOp { + id @0 :Common.Id; + # The async op's ID. + # + # This uniquely identifies this op across all *currently live* ones. + + metadata @1 :Common.MetaId; + # The numeric id of the op's Metadata + # + # This identifies the `Metadata` that describes the `tracing` span + # corresponding to this async op. The metadata for this ID will have been sent + # in a prior `RegisterMetadata` message. + + source @2 :Text; + # The source of this async operation. Most commonly this should be the name of the method where + # the instantiation of this op has happened. + + parentAsyncOp @3 :Common.Id; + # The ID of the parent async op. + # + # This field is only set if this async op was created while inside of another async op. + + resourceId @4 :Common.Id; + # The resource's ID +} + +struct Stats { + createdAt @0 :Common.Timestamp; + droppedAt @1 :Common.Timestamp; + + task @2 :Common.Id; + # The ID of the task currently awaiting on this op + + pollStats @3 :Common.PollStats; + + attributes @4 :List(Common.Attribute); +} diff --git a/runtime/console-api/schema/common.capnp b/runtime/console-api/schema/common.capnp new file mode 100644 index 0000000..60d9d7a --- /dev/null +++ b/runtime/console-api/schema/common.capnp @@ -0,0 +1,228 @@ +@0x911648bc10d7e94f; + +struct Timestamp { + # Timestamp in consisting of a 64-bit second counter and a 32-bit nanosecond counter + + seconds @0 :Int64; + # Non-leap second counter since 1970-01-01T00:00:00Z + + nanoseconds @1 :UInt32; + # Nanosecond counter since the last non-leap second +} + +struct Duration { + # Duration struct consiting of a 64-bit seconds value and a 32-bit *signed* nanosecond value + + seconds @0 :Int64; + # Seconds of duration + + nanoseconds @1 :Int32; + # Nanoseconds of duration +} + +struct Optional(T) { + union { + nothing @0 :Void; + just @1 :T; + } +} + +struct Id { + id @0 :UInt64; +} + +struct MetaId { + id @0 :UInt64; +} + +struct SpanId { + id @0 :UInt64; +} + +struct Field { + name :union { + string @0 :Text; + # Name as immediate string + index @1 :UInt64; + # An index position into the `Metadata.field_names` of the metadata for + # the task span that the field came from + } + + value :union { + debug @2 :Text; + # Value serialized to string using `fmt::Debug` + string @3 :Text; + # A string value + u64 @4 :UInt64; + # An unsigned integer value + i64 @5 :Int64; + # A signed integer value + bool @6 :Bool; + # A boolean value + } + + metadataId @7 :MetaId; + # Metadata for the task span that the field came from +} + +struct Span { + # Represents a period of time in which a program was executing in a particular context. + # + # Corresponds to `Span` in the `tracing` crate. + + id @0 :SpanId; + # An id that unique identifies this span in relation to other spans + + metadataId @1 :MetaId; + # Identifier for metadata describing static characteristics of all spans originating + # from that callsite, such as its name, source code location, verbosity level, and + # the names of its fields. + + fields @2 :List(Field); + # User-defined key-value pairs of arbitrary data that describe the context the span represents + + at @3 :Timestamp; + # Timestamp for the span +} + +struct Location { + file @0 :Text; + # File path + modulePath @1 :Text; + # Rust module path + line @2 :UInt32; + # Line number + column @3 :UInt32; +} + +struct RegisterMetadata { + # Any new metadata that was registered since the last update + + metadata @0 :List(NewMetadata); + # The new metadata that was registered + + struct NewMetadata { + id @0 :MetaId; + # Unique identifier for the metadata + + metadata @1 :Metadata; + # Metadata payload + } +} + +struct Metadata { + # Metadata associated with a span or an event + + name @0 :Text; + # Name of the span or event + + target @1 :Text; + # Describes the part of the system where the span or event that this metadata describes occured + + modulePath @2 :Text; + # Path to the Rust module where the span occured + + location @3 :Location; + # Rust source code location associated with this span or event + + kind @4 :Kind; + # Indicates wheter this metadata is associated with a span or an event + enum Kind { + span @0; + event @1; + } + + level @5 :Level; + # Describes the level of verbosity of a span or event + enum Level { + error @0; + # Designates very serious errors + + warn @1; + # Designates hazardous situations + + info @2; + # Designates useful information + + debug @3; + # Designates lower priority information + + trace @4; + # Designates very low priority, often extremely verbose, information + } +} + +struct PollStats { + # Contains stats about objects that can be polled. Currently these can be: + # - tasks that have been spawned + # - async operations that are performed within the context of a task + + polls @0 :UInt64; + # The total number of times this object was polled + + firstPoll @1 :Optional(Timestamp); + # The timestamp of the first time this object was polled. + # + # If this is `Nothing`, the object has not yet been polled. + # + # Subtracting this timestamp from `created_at` can be used to calculate the + # time to first poll for this object, a measurement of executor latency. + + lastPollStarted @2 :Optional(Timestamp); + # The timestamp of the most recent time this objects's poll method was invoked. + # + # If this is `Nothing`, the object has not yet been polled. + # + # If the object has only been polled a single time, then this value may be + # equal to the `first_poll` timestamp. + + lastPollEnded @3 :Optional(Timestamp); + # The timestamp of the most recent time this objects's poll method finished execution. + # + # If this is `Nothing`, the object has not yet been polled or is currently being polled. + # + # If the object does not exist anymore, then this is the time the final invocation of + # its poll method has completed. + + busyTime @4 :Duration; + # The total duration this object was being *actively polled*, summed across + # all polls. + # + # Note that this includes only polls that have completed, and does not + # reflect any in-progress polls. Subtracting `busy_time` from the + # total lifetime of the polled object results in the amount of time it + # has spent *waiting* to be polled (including the `scheduled_time` value + # from `TaskStats`, if this is a task). +} + +struct Attribute { + # State attributes of an entity. These are dependent on the type of the entity. + # + # For example, a timer resource will have a duration, while a semaphore resource may + # have a permit count. Likewise, the async ops of a semaphore may have attributes + # indicating how many permits they are trying to acquire vs how many are acquired. + # These values may change over time. Therefore, they live in the runtime stats rather + # than the static data describing the entity. + + field @0 :Field; + # The key-value pair for the attribute + + unit @1 :Text; + # Some values carry an unit of measurement, e.g. durations given in 'ms'. +} + +struct Map(K, V) { + entries @0 :List(Entry); + struct Entry { + key @0 :K; + value @1 :V; + } +} + +struct IntMap(V) { + entries @0 :List(Entry); + struct Entry { + key @0 :UInt64; + value @1 :V; + } +} diff --git a/runtime/console-api/schema/instrument.capnp b/runtime/console-api/schema/instrument.capnp new file mode 100644 index 0000000..6a004fe --- /dev/null +++ b/runtime/console-api/schema/instrument.capnp @@ -0,0 +1,50 @@ +@0xd53fadcd6c8f437f; + +using Common = import "common.capnp"; +using Stream = import "stream.capnp"; +using Tasks = import "tasks.capnp"; +using Resources = import "resources.capnp"; +using AsyncOps = import "async_ops.capnp"; + +interface Instrument { + watchUpdates @0 (request :InstrumentRequest, receiver :Stream.Receiver(Update)) -> (stream :Stream.Sender); + watchTaskDetails @1 (request :TaskDetailsRequest, receiver :Stream.Receiver(Tasks.TaskDetails)) -> (stream :Stream.Sender); + pause @2 PauseRequest -> PauseResponse; + resume @3 ResumeRequest -> ResumeResponse; +} + +struct InstrumentRequest { + +} + +struct Update { + now @0 :Common.Timestamp; + + taskUpdate @1 :Tasks.Update; + + resourceUpdate @2 :Resources.Update; + + asyncOpUpdate @3 :AsyncOps.Update; + + newMetadata @4 :Common.RegisterMetadata; +} + +struct TaskDetailsRequest { + id @0 :Common.Id; +} + +struct PauseRequest { + +} + +struct PauseResponse { + +} + +struct ResumeRequest { + +} + +struct ResumeResponse { + +} diff --git a/runtime/console-api/schema/resources.capnp b/runtime/console-api/schema/resources.capnp new file mode 100644 index 0000000..82e7f9f --- /dev/null +++ b/runtime/console-api/schema/resources.capnp @@ -0,0 +1,98 @@ +@0xd75a34c00d06ab36; + +using Common = import "common.capnp"; + +struct Update { + newResource @0 :List(Resource); + # List of new resources that were created since the last Update was sent + + statsUpdate @1 :Common.IntMap(Stats); + # All resource stats that have changed since the last update + + newPollOps @2 :List(PollOp); + # List of all new poll ops that have been invoked on resources since the last update + + droppedEvents @3 :UInt64; + # Count of how many resource events were not recorded because the applications event buffer was + # at capacity. + # + # If everything is working normally, this is 0. If it is not that may inidcate that some data is + # missing from this update and it may be necessary to increase the number of events buffered by + # the application to ensure that data loss is avoided. + # + # If the application's instrumentation ensures reliable delivery of events, this will alway be 0 +} + +struct Resource { + id @0 :Common.Id; + # The resource's ID. + # + # This uniquely identifies this resource across all *currently live* resoures. + + metadata @1 :Common.MetaId; + # Numeric ID of the resource's Metadata. + + concreteType @2 :Text; + # Name of the concrete Rust type of this resource + + kind @3 :Kind; + # The kind of this resource + struct Kind { + union { + wellKnown @0 :WellKnown; + other @1 :Text; + } + + enum WellKnown { + timer @0; + } + } + + location @4 :Common.Location; + # The location in code where this resource was created + + parentResourceId @5 :Common.Id; + # ID of the parent resource + + internal @6 :Bool; + # Wether or not this resource is an internal component of another resource +} + +struct Stats { + # Task runtime stats of a resource + + createdAt @0 :Common.Timestamp; + # Timestamp of when the resource was created + + droppedAt @1 :Common.Timestamp; + # Timestamp of when the resource was dropped + + attributes @2 :List(Common.Attribute); + # State attributes of the resource. These are dependent on the type of the resource. For example + # a timer resource will have a duration, while a semaphore resource may have permits as an + # attribute. These values may change over time as the state of the resource changes. Therefore + # they live in the runtime stats rather than the static data describing the resource +} + +struct PollOp { + metadata @1 :Common.MetaId; + # Numeric ID of the op's Metadata + # + # This identifies the `Metadata` that describes the `tracing` span corresponding to this op. The + # metadata for this ID will have been sent in a prior `RegisterMetadata` message. + + resource @0 :Common.Id; + # The resource's ID + + name @2 :Text; + # Name of this op (e.g. poll_elapsed, new_timeout, reset, etc.) + + task @3 :Common.Id; + # The ID of the task context that this poll op has been called from + + asyncOp @4 :Common.Id; + # The ID of the async op that this poll op is part of + + ready @5 :Bool; + # Wheter this poll op has return with `Ready` or not. +} diff --git a/runtime/console-api/schema/stream.capnp b/runtime/console-api/schema/stream.capnp new file mode 100644 index 0000000..68f5dd3 --- /dev/null +++ b/runtime/console-api/schema/stream.capnp @@ -0,0 +1,12 @@ +@0x85894bd060182ed3; + +interface Receiver(T) { + send @0 (chunk :T) -> stream; + done @1 (); +} + +interface Sender { + pause @0 (); + resume @1 (); + stop @2 (); +} diff --git a/runtime/console-api/schema/tasks.capnp b/runtime/console-api/schema/tasks.capnp new file mode 100644 index 0000000..863aa11 --- /dev/null +++ b/runtime/console-api/schema/tasks.capnp @@ -0,0 +1,160 @@ +@0xef8effe12d55773f; + +using Common = import "common.capnp"; + +struct Task { + # Data recorded when a new task is spawned + + id @0 :Common.Id; + # The task's ID. + # + # This uniquely identifies this task across all *currently live* tasks. + # When the task's stats change, or when the task completes, it will be + # identified by this ID; if the client requires additional information + # included in the `Task` message, it should store that data and access it + # by ID. + + metadata @1 :Common.MetaId; + # The numeric ID of the task's `Metadata`. + # + # This identifies the `Metadata` that describes the `tracing` span + # corresponding to this task. The metadata for this ID will have been sent + # in a prior `RegisterMetadata` message. + + kind @2 :Kind; + # The category of task this task belongs to. + enum Kind { + spawn @0; + # Spawned using the main runtime asyncronous task spawning + blocking @1; + # Spawning as a blocking operation + } + + fields @3 :List(Common.Field); + # A list of 'fields' attached to this task. + + parents @4 :List(Common.SpanId); + # An ordered list of span IDs corresponding to the `tracing` span context + # in which this task was spawned. + # + # The first span ID in this list is the immediate parent, followed by that + # span's parent, and so on. The final ID is the root span of the current + # trace. + # + # If this is empty, there were *no* active spans when the task was spawned. + # + # These IDs may correspond to `tracing` spans which are *not* tasks, if + # additional trace data is being collected. + + location @5 :Common.Location; + # The location in the code where this task was spawned. +} + +struct Update { + # A task state update. + # + # Each `Update` contains any task data that has changed since the last + # update. This includes: + # - any new tasks that were spawned since the last update + # - the current stats for any task whose stats changed since the last update + + newTasks @0 :List(Task); + # A list of new tasks that were spawned since the last `Update` was + # sent. + # + # If this is empty, no new tasks were spawned. + + statsUpdate @1 :Common.IntMap(Stats); + # Any task stats that have changed since the last update. + # + # This is a map of task IDs (64-bit unsigned integers) to task stats. If a + # task's ID is not included in this map, then its stats have *not* changed + # since the last `Update` in which they were present. If a task's ID + # *is* included in this map, the corresponding value represents a complete + # snapshot of that task's stats at in the current time window. + + droppedEvents @2 :UInt64; + # A count of how many task events (e.g. polls, spawns, etc) were not + # recorded because the application's event buffer was at capacity. + # + # If everything is working normally, this should be 0. If it is greater + # than 0, that may indicate that some data is missing from this update, and + # it may be necessary to increase the number of events buffered by the + # application to ensure that data loss is avoided. + # + # If the application's instrumentation ensures reliable delivery of events, + # this will always be 0. +} + +struct TaskDetails { + # A task details update + + id @0 :Common.Id; + # The tasks id to which the details belong to + + now @1 :Common.Timestamp; + # The timestamp for when the update to the task took place + + pollTimesHistogram @2 :DurationHistogram; + # A histogram of task poll durations + + scheduledTimesHistogram @3 :DurationHistogram; + # A histogram of task scheduled durations + # + # The scheduled duration is the time a task spends between woken and when it is next polled +} + +struct Stats { + # Task performance statistics + + createdAt @0 :Common.Timestamp; + # Timestamp of when the task was created + + droppedAt @1 :Common.Timestamp; + # Timestamp of when the task was dropped + + wakes @2 :UInt64; + # Total number of times this task has been woken over its lifetime + + wakerClones @3 :UInt64; + # Total number of times this tasks waker was cloned + + wakerDrops @4 :UInt64; + # Total number of times this tasks waker was dropped + + lastWake @5 :Common.Optional(Common.Timestamp); + # The timestamp of the most recent time this task has been woken. + # + # If this is `Nothing`, the task has not yet been woken + + pollStats @6 :Common.PollStats; + # Contains task poll statistics + + selfWakes @7 :UInt64; + # Total number of times this task has woken itself + + scheduledTime @8 :Common.Duration; + # The total duration this task was scheduled prior to being polled, summed + # across all poll cycles. + # + # Note that this includes only polls that have started, and does not + # reflect any scheduled state where the task hasn't yet been polled. + # Subtracting both `busy_time` (from the task's `PollStats`) and + # `scheduled_time` from the total lifetime of the task results in the + # amount of time it spent unable to progress because it was waiting on + # some resource. +} + +struct DurationHistogram { + rawHistogram @0 :Data; + # HdrHistogram.rs `Histogram` serialized to binary in the V2 format + + maxValue @1 :UInt64; + # The histograms maximum value + + highOutliers @2 :UInt64; + # The number of outliers which have exceeded the histograms maximum value + + highestOutlier @3 :UInt64; + # The highest recorded outlier. This only has a sensible value if `highOutliers` is greater than zero +} diff --git a/runtime/console-api/schema/trace.capnp b/runtime/console-api/schema/trace.capnp new file mode 100644 index 0000000..aaf1a7a --- /dev/null +++ b/runtime/console-api/schema/trace.capnp @@ -0,0 +1,80 @@ +@0xb021b19fd5986342; + +using Common = import "common.capnp"; +using Stream = import "stream.capnp"; + +interface Trace { + watch @0 (request :WatchRequest, receiver :Stream.Receiver(TraceEvent)) -> (stream :Stream.Sender); +} + +struct WatchRequest { + # Start watching trace events with the provided filter + + filter @0 :Text; + # Filters which trace events should be streamed +} + +struct TraceEvent { + union { + registerThread @0 :RegisterThreads; + # A new thread was registered + + registerMetadata @1 :Common.RegisterMetadata; + # A new span metadata was registered + + newSpan @2 :Common.Span; + # A span was created + + enterSpan @3 :Enter; + # A span was entered + + exitSpan @4 :Exit; + # A span was exited + + closeSpan @5 :Close; + # A span was closed + } + + struct RegisterThreads { + # Signals that a new thread was registered + + names @0 :Common.IntMap(Text); + # Maps the registered thread id's to their associated name + } + + struct Enter { + # Signals that a span was entered + + spanId @0 :Common.SpanId; + # Identified the span that was entered + + threadId @1 :UInt64; + # Identifies who entered the span + + at @2 :Common.Timestamp; + # Identifies when the span was entered + } + + struct Exit { + # Signals that a span was exited + + spanId @0 :Common.SpanId; + # Identifies the spawn that was exited + + threadId @1 :UInt64; + # Identifies who exited the span + + at @2 :Common.Timestamp; + # Identifies when the span was exited + } + + struct Close { + # Signals that a span was closed + + spanId @0 :Common.SpanId; + # Identifies the span that was closed + + at @1 :Common.Timestamp; + # Identifies when the span was closed + } +} diff --git a/runtime/console-api/src/lib.rs b/runtime/console-api/src/lib.rs new file mode 100644 index 0000000..7d12d9a --- /dev/null +++ b/runtime/console-api/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +}