prepared_execution.h#

namespace onnx_light

Alias that makes onnx-light headers compatible with code that references ONNX_LIGHT_NAMESPACE (the macro used in the standard onnx package).

Set to ONNX_LIGHT_NAMESPACE so both names resolve to the same namespace.

Symbol-visibility attribute for the public onnx-light C++ API.

Maps the upstream compatibility macro to onnx-light’s explicit proto ABI annotation. This keeps declarations from vendored ONNX headers visible when lib_onnx_proto uses hidden visibility by default.

Namespace alias so that ONNX C++ code (and consumers such as onnxruntime) that refers to the literal onnx namespace — rather than the ONNX_NAMESPACE macro — resolves to the onnx-light namespace. The standard onnx package lives in namespace onnx; onnx-light uses onnx_light (via ONNX_LIGHT_NAMESPACE), so this alias keeps onnx-light a true drop-in. It is only introduced when the onnx-light namespace differs from onnx.

namespace core

namespace runtime

Typedefs

using PreparedTaskExecutor = std::function<void(const TaskDescriptor&, PreparedExecutionState&)>#

Enums

enum class PreparedResidencyState#

Values:

enumerator kAbsent#

enumerator kLoading#

enumerator kPreparing#

enumerator kResident#

enumerator kFailed#

enumerator kEvicting#

enumerator kPersisted#

enum class MaterializationRecipeKind#

Values:

enumerator kReadPackedPayload#

enumerator kReadSourceAndPrepack#

Functions

ONNX_LIGHT_CORE_API MaterializationTaskDescriptors ExpandMaterializationRecipe (const PreparedRequirementDescriptor &requirement, const MaterializationRecipe &selected, TaskId first_task_id): Expands one selected recipe into its session-scoped preparation task chain.

class ExecutionHandle#

#include <prepared_execution.h>

Owns the completion state of one PrepareAsync/RunAsync submission.

The handle is movable but not copyable. It keeps the submission’s borrowed PreparedExecutionState and task executor referenced by the caller alive for as long as the submission still runs: destroying (or move-assigning over) a live handle blocks until every task the submission still owns reaches a terminal state, so no detached task can outlive the context or input state its submission borrowed. Cancel() only affects tasks this submission produces; shared session-scoped work still required elsewhere is left untouched.

Public Functions

ExecutionHandle()#

ExecutionHandle(ExecutionHandle&&) noexcept#

ExecutionHandle &operator=(ExecutionHandle&&) noexcept#

ExecutionHandle(const ExecutionHandle&) = delete#

ExecutionHandle &operator=(const ExecutionHandle&) = delete#

~ExecutionHandle()#

const PreparedExecutionResult &Wait()#: Blocks until every required task finishes and rethrows the first error.

bool IsReady() const#: Returns whether the submission has already reached a terminal state.

void Cancel()#

Cooperatively cancels tasks this submission has not yet dispatched.

Already-running tasks finish normally; downstream tasks are suppressed. Session-scoped work owned by another submission is left untouched.

Private Functions

void WaitQuiet() noexcept#: Blocks until done without rethrowing; safe to call while unwinding.

Private Members

std::shared_ptr<Shared> shared_#

Private Static Functions

static ExecutionHandle Submit(std::function<PreparedExecutionResult(std::atomic<bool>&)> work)#

Friends

friend class PreparedExecutionPlan

struct MaterializationRecipe#

#include <prepared_execution.h>

Public Members

MaterializationRecipeKind kind = MaterializationRecipeKind::kReadSourceAndPrepack #

std::string payload_id#

std::string layout#

struct MaterializationTaskDescriptors#

#include <prepared_execution.h>

Public Members

TaskDescriptor load#

TaskDescriptor prepack#

TaskDescriptor publish#

TaskDescriptor dormant_fallback#

class PreparationArena : public onnx_light::core::runtime::ExecutionArena #

#include <prepared_execution.h>

Public Functions

virtual RawBuffer *Allocate(size_t n_bytes) override#

Allocates n_bytes from the smallest sufficient retained buffer.

An unused slot is preferred when no retained buffer is large enough. Once all slots have acquired storage, the largest undersized free buffer grows to satisfy the request.

Throws:: std::bad_alloc – if all slots are currently live.

virtual void Free(RawBuffer *buffer) override#

Returns a live buffer to its capacity bucket without releasing its storage.

Throws:: std::invalid_argument – if buf is not live in this arena.

virtual size_t TotalAllocatedSize() const override#: Returns the logical bytes held by live buffers.

virtual size_t PeakAllocatedSize() const override#: Returns the peak logical live-byte count.

virtual void ResetPeak() override#: Resets the live-byte peak to the current live-byte count.

explicit ExecutionArena(size_t capacity, size_t retention_cap = std::numeric_limits<size_t>::max())#

Constructs an arena with at most capacity simultaneously live buffers.

Parameters:

capacity – Number of stable :cpp:struct:RawBuffer slots.
retention_cap – Maximum total capacity kept on the retained free lists. Defaults to unbounded, disabling eviction.

Private Members

mutable std::mutex mutex_#

class PreparedArena : public onnx_light::core::runtime::ExecutionArena #

#include <prepared_execution.h>

Public Functions

virtual RawBuffer *Allocate(size_t n_bytes) override#

Allocates n_bytes from the smallest sufficient retained buffer.

An unused slot is preferred when no retained buffer is large enough. Once all slots have acquired storage, the largest undersized free buffer grows to satisfy the request.

Throws:: std::bad_alloc – if all slots are currently live.

virtual void Free(RawBuffer *buffer) override#

Returns a live buffer to its capacity bucket without releasing its storage.

Throws:: std::invalid_argument – if buf is not live in this arena.

virtual size_t TotalAllocatedSize() const override#: Returns the logical bytes held by live buffers.

virtual size_t PeakAllocatedSize() const override#: Returns the peak logical live-byte count.

virtual void ResetPeak() override#: Resets the live-byte peak to the current live-byte count.

explicit ExecutionArena(size_t capacity, size_t retention_cap = std::numeric_limits<size_t>::max())#

Constructs an arena with at most capacity simultaneously live buffers.

Parameters:

capacity – Number of stable :cpp:struct:RawBuffer slots.
retention_cap – Maximum total capacity kept on the retained free lists. Defaults to unbounded, disabling eviction.

Private Members

mutable std::mutex mutex_#

class PreparedExecutionPlan#

#include <prepared_execution.h>

Describes session preparation and invocation execution in one immutable graph.

Public Functions

explicit PreparedExecutionPlan(std::vector<TaskDescriptor> tasks)#

PreparedExecutionPlan(std::vector<TaskDescriptor> tasks, const RequiredPayloadManifest &payload_manifest)#

PreparedExecutionPlan(std::vector<TaskDescriptor> invocation_tasks, std::vector<PreparedMaterialization> materializations, const RequiredPayloadManifest &payload_manifest)#

inline const std::vector<TaskDescriptor> &tasks() const noexcept#

PreparedExecutionResult RunSequential(PreparedExecutionState &state, const PreparedTaskExecutor &executor) const#

Executes session producers and the invocation tasks through completion events.

Parameters:

state – Mutable session state that owns reusable session-task generations.
executor – Executes a task descriptor. It may be called concurrently by the session worker and the invoking thread.

Returns:

Fresh invocation diagnostics and the session generations it observed.

PreparedExecutionResult RunParallel(PreparedExecutionState &state, const PreparedTaskExecutor &executor, CpuExecutor &cpu_executor) const#

Executes ready tasks using persistent bounded I/O workers and cpu_executor.

CPU work is dispatched only through the supplied session executor; this method never creates another CPU worker pool.

ExecutionHandle PrepareAsync(PreparedExecutionState &state, const PreparedTaskExecutor &executor, CpuExecutor *cpu_executor = nullptr) const#

Submits only this plan’s session-scoped tasks and returns immediately.

state, executor, and (when supplied) cpu_executor must outlive the returned handle.

ExecutionHandle RunAsync(PreparedExecutionState &state, const PreparedTaskExecutor &executor, CpuExecutor *cpu_executor = nullptr) const#

Submits every task in this plan and returns immediately.

This plan, state, executor, and (when supplied) cpu_executor must all outlive the returned handle.

void Prepare(PreparedExecutionState &state, const PreparedTaskExecutor &executor, CpuExecutor *cpu_executor = nullptr) const#: Synchronous PrepareAsync followed by Wait.

Private Functions

PreparedExecutionResult Run(PreparedExecutionState &state, const PreparedTaskExecutor &executor, CpuExecutor *cpu_executor, const std::atomic<bool> *cancel_requested = nullptr) const#

Private Members

std::vector<TaskDescriptor> tasks_#

std::vector<PreparedKey> prepared_requirements_#

struct PreparedExecutionResult#

#include <prepared_execution.h>

Public Members

uint64_t invocation_id = 0#

std::vector<TaskDiagnostic> diagnostics#

std::vector<std::pair<TaskId, uint64_t>> session_generations#

bool used_hot_path = false#

size_t enqueued_tasks = 0#

size_t continuation_suspensions = 0#

size_t peak_in_flight_bytes = 0#

std::vector<PreparedTaskTrace> trace#

class PreparedExecutionState#

#include <prepared_execution.h>

Owns session preparation arenas and the mutable prepared-object residency state.

Public Functions

explicit PreparedExecutionState(size_t preparation_slots = 4, size_t prepared_slots = 4, size_t preparation_retention_cap = std::numeric_limits<size_t>::max(), size_t prepared_retention_cap = std::numeric_limits<size_t>::max(), PreparedSchedulerOptions scheduler_options = {})#

~PreparedExecutionState()#

inline PreparationArena &preparation_arena() noexcept#

inline PreparedArena &prepared_arena() noexcept#

AllocationHandle AllocatePrepared(size_t n_bytes)#

inline PreparedObjectStore &objects() noexcept#

inline const PreparedObjectStore &objects() const noexcept#

inline uint64_t readiness_epoch() const noexcept#

Private Functions

SessionTaskRequest RequestSessionTask(TaskId task_id, bool force_retry)#

Private Members

PreparationArena preparation_arena_#

PreparedArena prepared_arena_#

PreparedObjectStore objects_#

std::mutex session_tasks_mutex_#

std::unordered_map<uint64_t, std::unique_ptr<SessionTaskEntry>> session_tasks_#

std::atomic<uint64_t> next_invocation_id_ = {0}#

std::unique_ptr<SchedulerState> scheduler_#

std::mutex hot_path_mutex_#

std::shared_mutex residency_run_mutex_#

const PreparedExecutionPlan *hot_path_plan_ = nullptr#

uint64_t hot_path_epoch_ = 0#

Friends

friend class PreparedExecutionPlan

struct SessionTaskRequest#

Public Members

uint64_t generation = 0#

TaskCompletion completion = {TaskId{}}#

bool producer = false#

struct PreparedKeyHash#

#include <prepared_execution.h>

Public Functions

size_t operator()(const PreparedKey &key) const noexcept#

struct PreparedMaterialization#

#include <prepared_execution.h>

Public Members

PreparedRequirementDescriptor requirement#

MaterializationRecipe selected#

TaskId first_task_id#

TaskPriority priority = TaskPriority::kPrefetch #

struct PreparedObjectRequest#

#include <prepared_execution.h>

Public Members

PreparedKey key#

uint64_t generation = 0#

TaskCompletion completion#

bool producer = false#

struct PreparedObjectRequirement#

#include <prepared_execution.h>

Public Members

PreparedKey key#

std::string source_fallback#

class PreparedObjectStore#

#include <prepared_execution.h>

Tracks immutable prepared objects and shares one in-flight generation per key.

Public Functions

explicit PreparedObjectStore(size_t residency_budget = std::numeric_limits<size_t>::max())#

~PreparedObjectStore()#

PreparedObjectRequest Request(const PreparedObjectRequirement &requirement)#: Returns the resident generation or elects one caller to produce a new one.

void MarkPreparing(const PreparedObjectRequest &request)#: Transitions the request selected as producer from loading to preparing.

void Publish(const PreparedObjectRequest &request, AllocationHandle allocation)#: Atomically installs a complete allocation before satisfying its completion.

void Fail(const PreparedObjectRequest &request, std::exception_ptr error, std::string message = {})#: Fails the current generation and preserves its diagnostic exception.

std::optional<PreparedObjectView> Find(const PreparedKey &key) const#: Returns a resident object’s immutable view, or no value when not resident.

bool Evict(const PreparedKey &key)#: Evicts a resident generation and returns its allocation to its owner.

void AdmitAllocation(size_t n_bytes)#: Makes room for an allocation before it consumes a prepared-arena slot.

PreparedResidencyState State(const PreparedKey &key) const#

size_t resident_bytes() const#

size_t waiting_admissions() const#

inline size_t residency_budget() const noexcept#

inline uint64_t readiness_epoch() const noexcept#

Private Members

std::unordered_map<PreparedKey, std::unique_ptr<Entry>, PreparedKeyHash> entries_#

size_t residency_budget_#

size_t resident_bytes_ = 0#

size_t waiting_admissions_ = 0#

mutable uint64_t access_epoch_ = 0#

std::shared_ptr<ResidencyTracker> residency_tracker_#

std::atomic<uint64_t> readiness_epoch_ = {0}#

Private Static Functions

static void ValidateGeneration(const Entry &entry, const PreparedObjectRequest &request)#

struct PreparedObjectView#

#include <prepared_execution.h>

Public Members

const RawBuffer *buffer = nullptr#

RawBufferAllocator *owner = nullptr#

uint64_t generation = 0#

std::shared_ptr<const AllocationHandle> pin#

struct PreparedRequirementDescriptor#

#include <prepared_execution.h>

Public Members

PreparedObjectRequirement requirement#

std::vector<MaterializationRecipe> recipes#

struct PreparedSchedulerOptions#

#include <prepared_execution.h>

Public Members

size_t io_workers = 2#

size_t global_memory_budget = std::numeric_limits<size_t>::max()#

size_t preparation_memory_budget = std::numeric_limits<size_t>::max()#

size_t prepared_memory_budget = std::numeric_limits<size_t>::max()#

size_t execution_memory_budget = std::numeric_limits<size_t>::max()#

size_t io_memory_budget = std::numeric_limits<size_t>::max()#

size_t reserved_critical_memory = 0#

struct PreparedTaskTrace#

#include <prepared_execution.h>

Public Members

TaskId task_id#

TaskScope scope = TaskScope::kInvocation #

TaskKind kind = TaskKind::kExecute #

ResourceClass resource = ResourceClass::kCpu #

TaskPriority priority = TaskPriority::kPrefetch #

uint64_t start_ns = 0#

uint64_t end_ns = 0#