Skip to content

Add types to support checked output files #135

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: 10-05-add_and_expose_get_current_test_
Choose a base branch
from
+161 −82
Draft

Add types to support checked output files #135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
146 changes: 80 additions & 66 deletions core/Store.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ module P = Promise
(* All the data we need to handle the files that contain the captured output
for a test after applying all defaults and options. *)
type capture_paths = {
(* Human-friendly name: "stdout", "stderr", or "stdxxx" *)
standard_name : string;
(* Human-friendly name: "stdout" or the basename of user-specified file
path. *)
short_name : string;
Expand Down Expand Up @@ -174,7 +172,7 @@ let write_name_file (test : T.test) =

let must_create_expectation_workspace_for_test (test : T.test) =
let uses_internal_storage (x : T.checked_output_options) =
match x.expected_output_path with
match x.snapshot_path with
| None -> true
| Some _user_provided_path -> false
in
Expand Down Expand Up @@ -251,21 +249,29 @@ let get_orig_output_suffix (test : T.test) =
| [] -> None
| _ -> Some orig_suffix

let get_expected_output_path (test : T.test) default_name
let get_snapshot_path (test : T.test) default_name
(options : T.checked_output_options) =
match options.expected_output_path with
match options.snapshot_path with
| None -> get_expectation_workspace () / test.id / default_name
| Some path -> path

let get_file_snapshot_path (test : T.test) (x : T.checked_output_file) =
match x.options.snapshot_path with
| None -> get_expectation_workspace () / test.id / x.name
| Some path -> path

let short_name_of_checked_output_options default_name
(options : T.checked_output_options) =
match options.expected_output_path with
match options.snapshot_path with
| None -> default_name
| Some path -> Fpath.basename path

let get_output_path (test : T.test) filename =
get_status_workspace () / test.id / filename

let get_output_file_path (test : T.test) (x : T.checked_output_file) =
get_status_workspace () / test.id / ("file-" ^ x.name)

let get_exception_path (test : T.test) = get_output_path test "exception"

let store_exception (test : T.test) opt_msg =
Expand All @@ -287,68 +293,76 @@ let get_exception (test : T.test) =
let capture_paths_of_test (test : T.test) : capture_paths list =
let unchecked_paths =
{
standard_name = unchecked_filename;
short_name = unchecked_filename;
path_to_expected_output = None;
path_to_output = get_output_path test unchecked_filename;
}
in
match test.checked_output with
| Ignore_output -> [ unchecked_paths ]
| Stdout options ->
[
{
standard_name = stdout_filename;
short_name =
short_name_of_checked_output_options stdout_filename options;
path_to_expected_output =
Some (get_expected_output_path test stdout_filename options);
path_to_output = get_output_path test stdout_filename;
};
unchecked_paths;
]
| Stderr options ->
[
{
standard_name = stderr_filename;
short_name =
short_name_of_checked_output_options stderr_filename options;
path_to_expected_output =
Some (get_expected_output_path test stderr_filename options);
path_to_output = get_output_path test stderr_filename;
};
unchecked_paths;
]
| Stdxxx options ->
[
{
standard_name = stdxxx_filename;
short_name =
short_name_of_checked_output_options stdxxx_filename options;
path_to_expected_output =
Some (get_expected_output_path test stdxxx_filename options);
path_to_output = get_output_path test stdxxx_filename;
};
]
| Split_stdout_stderr (stdout_options, stderr_options) ->
[
{
standard_name = stdout_filename;
short_name =
short_name_of_checked_output_options stdout_filename stdout_options;
path_to_expected_output =
Some (get_expected_output_path test stdout_filename stdout_options);
path_to_output = get_output_path test stdout_filename;
};
{
standard_name = stderr_filename;
short_name =
short_name_of_checked_output_options stderr_filename stderr_options;
path_to_expected_output =
Some (get_expected_output_path test stderr_filename stderr_options);
path_to_output = get_output_path test stderr_filename;
};
]
let std_output_paths =
match test.checked_output with
| Ignore_output -> [ unchecked_paths ]
| Stdout options ->
[
{
short_name =
short_name_of_checked_output_options stdout_filename options;
path_to_expected_output =
Some (get_snapshot_path test stdout_filename options);
path_to_output = get_output_path test stdout_filename;
};
unchecked_paths;
]
| Stderr options ->
[
{
short_name =
short_name_of_checked_output_options stderr_filename options;
path_to_expected_output =
Some (get_snapshot_path test stderr_filename options);
path_to_output = get_output_path test stderr_filename;
};
unchecked_paths;
]
| Stdxxx options ->
[
{
short_name =
short_name_of_checked_output_options stdxxx_filename options;
path_to_expected_output =
Some (get_snapshot_path test stdxxx_filename options);
path_to_output = get_output_path test stdxxx_filename;
};
]
| Split_stdout_stderr (stdout_options, stderr_options) ->
[
{
short_name =
short_name_of_checked_output_options stdout_filename stdout_options;
path_to_expected_output =
Some (get_snapshot_path test stdout_filename stdout_options);
path_to_output = get_output_path test stdout_filename;
};
{
short_name =
short_name_of_checked_output_options stderr_filename stderr_options;
path_to_expected_output =
Some (get_snapshot_path test stderr_filename stderr_options);
path_to_output = get_output_path test stderr_filename;
};
]
in
let output_file_paths =
test.checked_output_files
|> Helpers.list_map (fun (x : T.checked_output_file) ->
{
short_name = x.name;
path_to_expected_output =
Some (get_file_snapshot_path test x);
path_to_output = get_output_file_path test x;
}
)
in
std_output_paths @ output_file_paths

let describe_unchecked_output (output : T.checked_output_kind) : string option =
match output with
Expand Down Expand Up @@ -386,15 +400,15 @@ let get_unchecked_output (test : T.test) =
| Error _cant_read_file -> None)
| None -> None

let get_expected_output_paths (paths : capture_paths list) =
let get_snapshot_paths (paths : capture_paths list) =
paths |> List.filter_map (fun x -> x.path_to_expected_output)

let get_expected_output (paths : capture_paths list) =
paths |> get_expected_output_paths |> list_map read_file
paths |> get_snapshot_paths |> list_map read_file

let set_expected_output (test : T.test) (capture_paths : capture_paths list)
(data : string list) =
let paths = capture_paths |> get_expected_output_paths in
let paths = capture_paths |> get_snapshot_paths in
if List.length data <> List.length paths then
Error.invalid_arg ~__LOC__
(sprintf "Store.set_expected_output: test %s, data:%i, paths:%i" test.name
Expand Down
2 changes: 0 additions & 2 deletions core/Store.mli
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,6 @@ val init_workspace : unit -> unit
for a test after applying all defaults and options.
*)
type capture_paths = {
(* Human-friendly name: "stdout", "stderr", or "stdxxx" *)
standard_name : string;
(* Human-friendly name: "stdout" or the basename of user-specified file
path. *)
short_name : string;
Expand Down
36 changes: 30 additions & 6 deletions core/Testo.ml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,9 @@ type subcommand_result = Cmd.subcommand_result =
module Promise = Promise
module Tag = Tag

(* abstract types *)
type checked_output_kind = T.checked_output_kind
type checked_output_file = T.checked_output_file

type t = T.test = {
id : string;
Expand All @@ -91,6 +93,7 @@ type t = T.test = {
func : unit -> unit Promise.t;
broken : string option;
checked_output : checked_output_kind;
checked_output_files : checked_output_file list;
expected_outcome : expected_outcome;
max_duration : float option;
normalize : (string -> string) list;
Expand All @@ -111,20 +114,35 @@ type alcotest_test = string * alcotest_test_case list
(* Conversions *)
(****************************************************************************)

(* TODO: rename expected_stdout_path -> snapshot_path *)
let stdout ?expected_stdout_path () : T.checked_output_kind =
Stdout { expected_output_path = expected_stdout_path }
Stdout { snapshot_path = expected_stdout_path }

let stderr ?expected_stderr_path () : T.checked_output_kind =
Stderr { expected_output_path = expected_stderr_path }
Stderr { snapshot_path = expected_stderr_path }

let stdxxx ?expected_stdxxx_path () : T.checked_output_kind =
Stdxxx { expected_output_path = expected_stdxxx_path }
Stdxxx { snapshot_path = expected_stdxxx_path }

let split_stdout_stderr ?expected_stdout_path ?expected_stderr_path () :
T.checked_output_kind =
Split_stdout_stderr
( { expected_output_path = expected_stdout_path },
{ expected_output_path = expected_stderr_path } )
( { snapshot_path = expected_stdout_path },
{ snapshot_path = expected_stderr_path } )

let validate_name =
let rex = Re.Pcre.regexp {|\A[a-zA-Z0-9_-]+\z|} in
fun str ->
Re.Pcre.pmatch ~rex str

let checked_output_file ?snapshot_path name : checked_output_file =
if not (validate_name name) then
invalid_arg
("Invalid 'name' argument for Testo.checked_output_file: " ^ name);
{
name;
options = { snapshot_path };
}

(*
Create an hexadecimal hash that is just long enough to not suffer from
Expand All @@ -145,6 +163,7 @@ let update_id (test : t) =
{ test with id; internal_full_name }

let create ?broken ?(category = []) ?(checked_output = T.Ignore_output)
?(checked_output_files = [])
?(expected_outcome = Should_succeed) ?max_duration ?(normalize = []) ?skipped ?solo
?(tags = []) ?(tolerate_chdir = false) ?tracking_url name func =
{
Expand All @@ -155,6 +174,7 @@ let create ?broken ?(category = []) ?(checked_output = T.Ignore_output)
func;
broken;
checked_output;
checked_output_files;
expected_outcome;
max_duration;
normalize;
Expand All @@ -166,9 +186,12 @@ let create ?broken ?(category = []) ?(checked_output = T.Ignore_output)
}
|> update_id

(* Update a setting if the new value 'option' is not None *)
let opt option default = Option.value option ~default

let update ?broken ?category ?checked_output ?expected_outcome ?func
let update
?broken ?category ?checked_output ?checked_output_files
?expected_outcome ?func
?max_duration ?normalize
?name ?skipped ?solo ?tags ?tolerate_chdir ?tracking_url old =
{
Expand All @@ -180,6 +203,7 @@ let update ?broken ?category ?checked_output ?expected_outcome ?func
(* requires same type for func and old.func *)
broken = opt broken old.broken;
checked_output = opt checked_output old.checked_output;
checked_output_files = opt checked_output_files old.checked_output_files;
expected_outcome = opt expected_outcome old.expected_outcome;
max_duration = opt max_duration old.max_duration;
normalize = opt normalize old.normalize;
Expand Down
35 changes: 33 additions & 2 deletions core/Testo.mli
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,24 @@ val split_stdout_stderr :
checked_output_kind
(** Same as {!val:stdxxx} but keep stdout and stderr separate. *)

type checked_output_file
(** A test's output file whose contents should be checked and reported
when it changes. *)

val checked_output_file :
?snapshot_path:Fpath.t ->
string ->
checked_output_file
(** [checked_output_file name] creates the specification for a checked output
file identified by the name [name]. Popular names include ["results.txt"]
and ["results.json"].
[name] must be a nonempty sequence of ASCII letters, digits,
underscores, dashes, or periods. Periods are not allowed in first or last
position. It is used by Testo in messaging and in file names.
The [snapshot_path] option specifies an alternate location for
the snapshot file that serves as the expectation for future test runs.
*)

module Promise : module type of Promise
(** Wrapper allowing for asynchronous test functions (Lwt and such). *)

Expand Down Expand Up @@ -140,14 +158,23 @@ type t = private {
command-line option causes the broken status to be ignored i.e.
a test run will fail if a broken test fails. *)
checked_output : checked_output_kind;
checked_output_files : checked_output_file list;
(** Files created by the test function whose contents must match
a reference snapshot.
The test function must copy its checked output files into
Testo's workspace using {!stash_output_files}. *)
expected_outcome : expected_outcome;
max_duration : float option;
(** A time limit for the test when running in a detached worker,
in seconds. This setting is ignored when running tests sequentially
in the master process such as with [-j0] or in [solo] mode. *)
normalize : (string -> string) list;
(** An optional function to rewrite any output data so as to mask the
variable parts. *)
variable parts. This is normalization is only applied to captured
stdout or stderr, not to output files. If you wish to normalize
output files, the test function should handle it, possibly with
the provided {!map_file} function.
*)
skipped : string option;
(** If not [None], the [skipped] property causes a test to be skipped
by Alcotest but still shown as ["[SKIP]"] rather than being
Expand Down Expand Up @@ -213,6 +240,7 @@ val create :
?broken:string ->
?category:string list ->
?checked_output:checked_output_kind ->
?checked_output_files:checked_output_file list ->
?expected_outcome:expected_outcome ->
?max_duration:float ->
?normalize:(string -> string) list ->
Expand All @@ -232,8 +260,10 @@ val create :
can be nested further using {!categorize} or {!categorize_suites}.}
{- [checked_output]: determines how to capture the test's output. Defaults
to no capture.}
{- [checked_output_files]: specifies the test's output files that should
remain the same from one test run to another.}
{- [expected_outcome]: whether a test is expected to complete without
raising an exception (default) or by raising an exception. }
raising an exception (default) or by raising an exception.}
{- [max_duration]: a time limit to run the test, in seconds.
It is honored only in tests running in workers i.e. not with the [-j0]
option of the test program.}
Expand All @@ -260,6 +290,7 @@ val update :
?broken:string option ->
?category:string list ->
?checked_output:checked_output_kind ->
?checked_output_files:checked_output_file list ->
?expected_outcome:expected_outcome ->
?func:(unit -> unit Promise.t) ->
?max_duration:float option ->
Expand Down
Loading