Rough Draft Implementation of Pipeline, PipelineJob, PipelineRun, and PipelineRunner

.clippy.toml

@@ -1,4 +1,4 @@
-excessive-nesting-threshold = 4
+excessive-nesting-threshold = 7
 too-many-arguments-threshold = 10
 allowed-idents-below-min-chars = ["..", "k", "f", "re", "id", "Ok", "'_"]
 allowed-duplicate-crates = [

.devcontainer/gpu/Dockerfile

@@ -23,7 +23,7 @@ RUN \
     apt-get clean
 
 ENV PATH="/root/.cargo/bin:${PATH}"
-RUN apt-get update && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y && \
+RUN apt-get update && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y --default-toolchain 1.86.0 && \
     # rust auto formatting
     rustup component add rustfmt && \
     # rust style linter

Cargo.toml

@@ -47,6 +47,7 @@ glob = "0.3.1"
 heck = "0.5.0"
 # random name generator
 names = "0.14.0"
+petgraph = { version = "0.8.1", features = ["serde-1"] }
 # complex pattern matching in strings
 regex = "1.11.0"
 # serialization/deserialization to/from filestore
@@ -96,6 +97,7 @@ as_conversions = { level = "allow", priority = 127 }                   # allow c
 assertions_on_result_states = { level = "allow", priority = 127 }      # allow checking is_ok/is_err
 big_endian_bytes = { level = "allow", priority = 127 }                 # allow to_be_bytes / from_be_bytes
 blanket_clippy_restriction_lints = { level = "allow", priority = 127 } # allow setting all restrictions so we can omit specific ones
+clone_on_ref_ptr = { level = "allow", priority = 127 }                 # allow clone on ref ptr
 default_numeric_fallback = { level = "allow", priority = 127 }         # allow type inferred by numeric literal
 disallowed_script_idents = { level = "allow", priority = 127 }         # skip since we use only ascii
 else_if_without_else = { level = "allow", priority = 127 }             # missing else ok
@@ -107,13 +109,16 @@ impl_trait_in_params = { level = "allow", priority = 127 }             # impl in
 implicit_return = { level = "allow", priority = 127 }                  # missing return ok
 inline_asm_x86_intel_syntax = { level = "allow", priority = 127 }      # intel syntax ok
 integer_division = { level = "allow", priority = 127 }                 # allow discarding remainder
+iter-over-hash-type = { level = "allow", priority = 127 }              # allow iterating over hash types
 little_endian_bytes = { level = "allow", priority = 127 }              # allow to_le_bytes / from_le_bytes
+min_ident_chars = { level = "allow", priority = 127 }                  # allow short identifiers
 missing_asserts_for_indexing = { level = "allow", priority = 127 }     # missing assert before indexing ok
 missing_docs_in_private_items = { level = "allow", priority = 127 }    # missing docs on private ok
 missing_inline_in_public_items = { level = "allow", priority = 127 }   # let rust compiler determine best inline logic
 missing_trait_methods = { level = "allow", priority = 127 }            # allow in favor of rustc `implement the missing item`
 module_name_repetitions = { level = "allow", priority = 127 }          # allow use of module name in type names
 multiple_inherent_impl = { level = "allow", priority = 127 }           # required in best practice to limit exposure over UniFFI
+multiple_crate_versions = { level = "allow", priority = 127 }          # allow multiple versions of the same crate
 must_use_candidate = { level = "allow", priority = 127 }               # omitting #[must_use] ok
 mod_module_files = { level = "allow", priority = 127 }                 # mod directories ok
 non_ascii_literal = { level = "allow", priority = 127 }                # non-ascii char in string literal ok
@@ -125,9 +130,11 @@ pub_use = { level = "allow", priority = 127 }                          # ok to s
 pub_with_shorthand = { level = "allow", priority = 127 }               # allow use of pub(super)
 pub_without_shorthand = { level = "allow", priority = 127 }            # allow use of pub(in super)
 question_mark_used = { level = "allow", priority = 127 }               # allow question operator
+result_large_err = { level = "allow", priority = 127 }                 # allow large error types, performance hit is acceptable
 self_named_module_files = { level = "allow", priority = 127 }          # mod files ok
 semicolon_inside_block = { level = "allow", priority = 127 }           # ok to keep inside block
 separated_literal_suffix = { level = "allow", priority = 127 }         # literal suffixes should be separated by underscore
+shadow_reuse = { level = "allow", priority = 127 }                     # Allow shadow reuses in the cases of impl into
 single_char_lifetime_names = { level = "allow", priority = 127 }       # single char lifetimes ok
 single_component_path_imports = { level = "allow", priority = 127 }    # allow for readability
 std_instead_of_alloc = { level = "allow", priority = 127 }             # we should use std when possible

src/core/crypto.rs

@@ -49,7 +49,7 @@ pub fn hash_buffer(buffer: impl AsRef<[u8]>) -> String {
 /// Will return error if unable to access file.
 pub fn hash_file(filepath: impl AsRef<Path>) -> Result<String> {
     hash_stream(
-        &mut File::open(&filepath).context(selector::InvalidFilepath {
+        &mut File::open(&filepath).context(selector::InvalidFileOrDirPath {
             path: filepath.as_ref(),
         })?,
     )
@@ -62,7 +62,10 @@ pub fn hash_file(filepath: impl AsRef<Path>) -> Result<String> {
 pub fn hash_dir(dirpath: impl AsRef<Path>) -> Result<String> {
     let summary: BTreeMap<String, String> = dirpath
         .as_ref()
-        .read_dir()?
+        .read_dir()
+        .context(selector::InvalidFileOrDirPath {
+            path: dirpath.as_ref(),
+        })?
         .map(|path| {
             let access_path = path?.path();
             Ok((

src/core/error.rs

@@ -9,6 +9,7 @@ use std::{
     io,
     path::{self},
 };
+use tokio::task::JoinError;
 
 impl From<BollardError> for OrcaError {
     fn from(error: BollardError) -> Self {
@@ -70,6 +71,17 @@ impl From<serde_yaml::Error> for OrcaError {
         }
     }
 }
+impl From<JoinError> for OrcaError {
+    fn from(error: JoinError) -> Self {
+        Self {
+            kind: Kind::IoError {
+                source: error.into(),
+                backtrace: Some(Backtrace::capture()),
+            },
+        }
+    }
+}
+
 impl From<Kind> for OrcaError {
     fn from(error: Kind) -> Self {
         Self { kind: error }
@@ -90,21 +102,26 @@ impl fmt::Debug for OrcaError {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         match &self.kind {
             Kind::EmptyResponseWhenLoadingContainerAltImage { backtrace, .. }
+            | Kind::FailedToExtractRunInfo { backtrace, .. }
+            | Kind::FailedToStartPod { backtrace, .. }
             | Kind::GeneratedNamesOverflow { backtrace, .. }
-            | Kind::InvalidFilepath { backtrace, .. }
+            | Kind::InvalidFileOrDirPath { backtrace, .. }
             | Kind::InvalidPodResultTerminatedDatetime { backtrace, .. }
             | Kind::KeyMissing { backtrace, .. }
             | Kind::NoAnnotationFound { backtrace, .. }
             | Kind::NoContainerNames { backtrace, .. }
+            | Kind::NodeNotFound { backtrace, .. }
             | Kind::NoFileName { backtrace, .. }
             | Kind::NoMatchingPodRun { backtrace, .. }
             | Kind::NoTagFoundInContainerAltImage { backtrace, .. }
+            | Kind::MissingStreamKey { backtrace, .. }
             | Kind::BollardError { backtrace, .. }
             | Kind::GlobPatternError { backtrace, .. }
             | Kind::IoError { backtrace, .. }
             | Kind::PathPrefixError { backtrace, .. }
             | Kind::SerdeJsonError { backtrace, .. }
-            | Kind::SerdeYamlError { backtrace, .. } => {
+            | Kind::SerdeYamlError { backtrace, .. }
+            | Kind::TokioJoinError { backtrace, .. } => {
                 write!(f, "{}{}", self.kind, format_stack(backtrace.as_ref()))
             }
         }

src/core/mod.rs

@@ -4,5 +4,9 @@ pub(crate) mod error;
 /// Components of the data model.
 pub mod model;
 pub(crate) mod orchestrator;
+/// Components relating to pipelines
+pub mod pipeline;
+/// Components relating to pipeline runner
+pub mod pipeline_runner;
 pub(crate) mod store;
 pub(crate) mod util;