diff --git a/docs/reference/testo-lwt/Testo_lwt/index.html b/docs/reference/testo-lwt/Testo_lwt/index.html index 178d2be..ad5745f 100644 --- a/docs/reference/testo-lwt/Testo_lwt/index.html +++ b/docs/reference/testo-lwt/Testo_lwt/index.html @@ -1,12 +1,18 @@ -Testo_lwt (testo-lwt.Testo_lwt)

Module Testo_lwt

Testo library - Utilities for writing OCaml test suites

Internal types

These types are documented in the library's source code in Types.ml. They are subject to frequent and unannounced changes at the whim of the library's authors. A casual user should not need them.

type expected_outcome =
  1. | Should_succeed
  2. | Should_fail of string
    (*

    explains why we expect this test to fail

    *)
type completion_status =
  1. | Test_function_returned
  2. | Test_function_raised_an_exception
  3. | Test_timeout
type fail_reason =
  1. | Raised_exception
  2. | Incorrect_output
  3. | Timeout
type outcome =
  1. | Succeeded
  2. | Failed of fail_reason
type captured_output =
  1. | Ignored of string
    (*

    unchecked combined output

    *)
  2. | Captured_stdout of string * string
    (*

    stdout, unchecked output

    *)
  3. | Captured_stderr of string * string
    (*

    stderr, unchecked output

    *)
  4. | Captured_stdout_stderr of string * string
    (*

    stdout, stderr

    *)
  5. | Captured_merged of string
    (*

    combined output

    *)
type expected_output =
  1. | Ignored
  2. | Expected_stdout of string
  3. | Expected_stderr of string
  4. | Expected_stdout_stderr of string * string
    (*

    stdout, stderr

    *)
  5. | Expected_merged of string
    (*

    combined output

    *)
type result = {
  1. completion_status : completion_status;
  2. captured_output : captured_output;
}
type missing_files =
  1. | Missing_files of Fpath.t list
type expectation = {
  1. expected_outcome : expected_outcome;
  2. expected_output : (expected_output, missing_files) Stdlib.Result.t;
}
type status = {
  1. expectation : expectation;
  2. result : (result, missing_files) Stdlib.Result.t;
}
type passing_status =
  1. | PASS
  2. | FAIL of fail_reason
  3. | XFAIL of fail_reason
  4. | XPASS
  5. | MISS of missing_files
type status_summary = {
  1. passing_status : passing_status;
  2. outcome : outcome;
  3. has_expected_output : bool;
}

Main interface

Test creation

type checked_output_kind

This type specifies what part of the output of a test (stdout, stderr) should be captured and compared against expectations.

Use the provided functions stdout, stderr, stdxxx, and split_stdout_stderr to create such an object.

val stdout : ?expected_stdout_path:Fpath.t -> unit -> checked_output_kind

Create an object of type checked_output_kind specifying that the test's standard output must be checked against a reference file.

val stderr : ?expected_stderr_path:Fpath.t -> unit -> checked_output_kind

Same as stdout but for capturing stderr instead.

val stdxxx : ?expected_stdxxx_path:Fpath.t -> unit -> checked_output_kind

Same as stdout but for capturing the combined stdout and stderr outputs.

val split_stdout_stderr : +Testo_lwt (testo-lwt.Testo_lwt)

Module Testo_lwt

Testo library - Utilities for writing OCaml test suites

Internal types

These types are documented in the library's source code in Types.ml. They are subject to frequent and unannounced changes at the whim of the library's authors. A casual user should not need them.

type expected_outcome =
  1. | Should_succeed
  2. | Should_fail of string
    (*

    explains why we expect this test to fail

    *)
type completion_status =
  1. | Test_function_returned
  2. | Test_function_raised_an_exception
  3. | Test_timeout
type fail_reason =
  1. | Raised_exception
  2. | Missing_output_file
  3. | Incorrect_output
  4. | Timeout
type outcome =
  1. | Succeeded
  2. | Failed of fail_reason
type checked_output_file

A test's output file whose contents captured from stdout or stderr should be checked and reported when it changes.

type checked_output_file_with_contents

A test's output file that is produced by the test function, checked, and reported when it changes.

type captured_output =
  1. | Ignored of string
    (*

    unchecked combined output

    *)
  2. | Captured_stdout of string * string
    (*

    stdout, unchecked output

    *)
  3. | Captured_stderr of string * string
    (*

    stderr, unchecked output

    *)
  4. | Captured_stdout_stderr of string * string
    (*

    stdout, stderr

    *)
  5. | Captured_merged of string
    (*

    combined output

    *)
type expected_output =
  1. | Ignored
  2. | Expected_stdout of string
  3. | Expected_stderr of string
  4. | Expected_stdout_stderr of string * string
    (*

    stdout, stderr

    *)
  5. | Expected_merged of string
    (*

    combined output

    *)
type result = {
  1. completion_status : completion_status;
  2. captured_output : captured_output;
  3. captured_output_files : checked_output_file_with_contents list;
  4. missing_output_files : Fpath.t list;
}
type missing_files =
  1. | Missing_files of Fpath.t list
type expectation = {
  1. expected_outcome : expected_outcome;
  2. expected_output : (expected_output, missing_files) Stdlib.Result.t;
  3. expected_output_files : (checked_output_file_with_contents, Fpath.t) + Stdlib.Result.t + list;
}
type status = {
  1. expectation : expectation;
  2. result : (result, missing_files) Stdlib.Result.t;
}
type passing_status =
  1. | PASS
  2. | FAIL of fail_reason
  3. | XFAIL of fail_reason
  4. | XPASS
  5. | MISS of missing_files
type status_summary = {
  1. passing_status : passing_status;
  2. outcome : outcome;
  3. has_expected_output : bool;
}

Main interface

Test creation

type checked_output_kind

This type specifies what part of the output of a test (stdout, stderr) should be captured and compared against expectations.

Use the provided functions stdout, stderr, stdxxx, and split_stdout_stderr to create such an object.

val stdout : ?expected_stdout_path:Fpath.t -> unit -> checked_output_kind

Create an object of type checked_output_kind specifying that the test's standard output must be checked against a reference file.

val stderr : ?expected_stderr_path:Fpath.t -> unit -> checked_output_kind

Same as stdout but for capturing stderr instead.

val stdxxx : ?expected_stdxxx_path:Fpath.t -> unit -> checked_output_kind

Same as stdout but for capturing the combined stdout and stderr outputs.

val split_stdout_stderr : ?expected_stdout_path:Fpath.t -> ?expected_stderr_path:Fpath.t -> unit -> - checked_output_kind

Same as stdxxx but keep stdout and stderr separate.

module Promise : module type of Promise

Wrapper allowing for asynchronous test functions (Lwt and such).

module Tag : module type of Testo_util.Tag

The type of tags which can be used to define subsets of tests precisely.

type t = private {
  1. id : string;
    (*

    Hash of the full name of the test, computed automatically.

    *)
  2. internal_full_name : string;
    (*

    Full name of the test, derived automatically from category and name.

    *)
  3. category : string list;
    (*

    Categories are made for organizing tests as a tree which is useful for display and filtering. A new category is created typically when grouping multiple test suites into one with 'categorize_suites' or when assigning a category to a list of tests with 'categorize'. e.g. ["food"; "fruit"; "kiwi"]

    *)
  4. name : string;
  5. func : unit -> unit Promise.t;
  6. broken : string option;
    (*

    If not None, the broken property causes the test to run normally but it will be ignored when determining the success of the test suite. This allows flaky tests to be kept around until they can be fixed. Use the string argument to explain briefly why the test is marked as broken. The --strict command-line option causes the broken status to be ignored i.e. a test run will fail if a broken test fails.

    *)
  7. checked_output : checked_output_kind;
  8. expected_outcome : expected_outcome;
  9. max_duration : float option;
    (*

    A time limit for the test when running in a detached worker, in seconds. This setting is ignored when running tests sequentially in the master process such as with -j0 or in solo mode.

    *)
  10. normalize : (string -> string) list;
    (*

    An optional function to rewrite any output data so as to mask the variable parts.

    *)
  11. skipped : string option;
    (*

    If not None, the skipped property causes a test to be skipped by Alcotest but still shown as "[SKIP]" rather than being omitted. The string should give a reason why the test is being skipped.

    *)
  12. solo : string option;
    (*

    If not None, this test will never run concurrently with other tests. The string should give a reason why the test should not run in parallel with other tests.

    *)
  13. tags : Tag.t list;
    (*

    Tags must be declared once using create_tag.

    *)
  14. tolerate_chdir : bool;
    (*

    If the test function changes the current directory without restoring it, it's an error unless this flag is set. All the tests in a test suite should share this field. To change the current directory temporarily and safely, use with_chdir.

    *)
  15. tracking_url : string option;
    (*

    A link to the relevant entry in a bug tracking system.

    *)
}

t is the type of a test. A test suite is a flat list of tests.

A test is at a minimum a name and a test function that raises exceptions to signal test failure. It is created with create or other similar functions provided by this module.

There are two main recommended ways of writing the test function:

1. With assert false:

Each test may use assert false to indicate that the test doesn't pass. This is the simplest way of failing while also showing the location of the failure. When using assert false, you should generally take care of printing the expected value and actual value to make debugging easier later.

2. With Alcotest.(check ...):

This is a little nicer because the error messages print something like "Expecting 'foo', got 'bar'". However, this can make tests slightly more complicated to write. If the test already prints the expected value and the actual value as its output, it's just easier to fail with assert false.

In any case, Alcotest will capture the output (stdout, stderr) of each test and put it in its own file so we can consult it later. Don't hesitate to log a lot during the execution of the test.

type test_with_status = t * status * status_summary
type subcommand_result =
  1. | Run_result of test_with_status list
  2. | Status_result of test_with_status list
  3. | Approve_result

The return type of each subcommand. It allows custom code to do something with the test data e.g. export to the JUnit format via the optional handle_subcommand_result argument of interpret_argv.

val create : + checked_output_kind

Same as stdxxx but keep stdout and stderr separate.

val checked_output_file : + ?snapshot_path:Fpath.t -> + string -> + checked_output_file

checked_output_file name creates the specification for a checked output file identified by the name name. Popular names include "results.txt" and "results.json". name must be a nonempty sequence of ASCII letters, digits, underscores, dashes, or periods. Periods are not allowed in first or last position. It is used by Testo in messaging and in file names. The snapshot_path option specifies an alternate location for the snapshot file that serves as the expectation for future test runs.

val stash_output_file : Fpath.t -> string -> unit

stash_output_file src_path dst_name copies a checked output file of the current test identified by its name dst_name into Testo's status workspace.

This function must be called by the test function for each checked output file.

module Promise : module type of Promise

Wrapper allowing for asynchronous test functions (Lwt and such).

module Tag : module type of Testo_util.Tag

The type of tags which can be used to define subsets of tests precisely.

type t = private {
  1. id : string;
    (*

    Hash of the full name of the test, computed automatically.

    *)
  2. internal_full_name : string;
    (*

    Full name of the test, derived automatically from category and name.

    *)
  3. category : string list;
    (*

    Categories are made for organizing tests as a tree which is useful for display and filtering. A new category is created typically when grouping multiple test suites into one with 'categorize_suites' or when assigning a category to a list of tests with 'categorize'. e.g. ["food"; "fruit"; "kiwi"]

    *)
  4. name : string;
  5. func : unit -> unit Promise.t;
  6. broken : string option;
    (*

    If not None, the broken property causes the test to run normally but it will be ignored when determining the success of the test suite. This allows flaky tests to be kept around until they can be fixed. Use the string argument to explain briefly why the test is marked as broken. The --strict command-line option causes the broken status to be ignored i.e. a test run will fail if a broken test fails.

    *)
  7. checked_output : checked_output_kind;
  8. checked_output_files : checked_output_file list;
    (*

    Files created by the test function whose contents must match a reference snapshot. The test function must copy its checked output files into Testo's workspace using stash_output_files.

    *)
  9. expected_outcome : expected_outcome;
  10. max_duration : float option;
    (*

    A time limit for the test when running in a detached worker, in seconds. This setting is ignored when running tests sequentially in the master process such as with -j0 or in solo mode.

    *)
  11. normalize : (string -> string) list;
    (*

    An optional function to rewrite any output data so as to mask the variable parts. This normalization is only applied to captured stdout or stderr, not to output files. If you wish to normalize output files, the test function should handle it, possibly with the provided map_file function.

    *)
  12. skipped : string option;
    (*

    If not None, the skipped property causes a test to be skipped by Alcotest but still shown as "[SKIP]" rather than being omitted. The string should give a reason why the test is being skipped.

    *)
  13. solo : string option;
    (*

    If not None, this test will never run concurrently with other tests. The string should give a reason why the test should not run in parallel with other tests.

    *)
  14. tags : Tag.t list;
    (*

    Tags must be declared once using create_tag.

    *)
  15. tolerate_chdir : bool;
    (*

    If the test function changes the current directory without restoring it, it's an error unless this flag is set. All the tests in a test suite should share this field. To change the current directory temporarily and safely, use with_chdir.

    *)
  16. tracking_url : string option;
    (*

    A link to the relevant entry in a bug tracking system.

    *)
}

t is the type of a test. A test suite is a flat list of tests.

A test is at a minimum a name and a test function that raises exceptions to signal test failure. It is created with create or other similar functions provided by this module.

There are two main recommended ways of writing the test function:

1. With assert false:

Each test may use assert false to indicate that the test doesn't pass. This is the simplest way of failing while also showing the location of the failure. When using assert false, you should generally take care of printing the expected value and actual value to make debugging easier later.

2. With Alcotest.(check ...):

This is a little nicer because the error messages print something like "Expecting 'foo', got 'bar'". However, this can make tests slightly more complicated to write. If the test already prints the expected value and the actual value as its output, it's just easier to fail with assert false.

In any case, Alcotest will capture the output (stdout, stderr) of each test and put it in its own file so we can consult it later. Don't hesitate to log a lot during the execution of the test.

type test_with_status = t * status * status_summary
type subcommand_result =
  1. | Run_result of test_with_status list
  2. | Status_result of test_with_status list
  3. | Approve_result

The return type of each subcommand. It allows custom code to do something with the test data e.g. export to the JUnit format via the optional handle_subcommand_result argument of interpret_argv.

val create : ?broken:string -> ?category:string list -> ?checked_output:checked_output_kind -> + ?checked_output_files:checked_output_file list -> ?expected_outcome:expected_outcome -> ?max_duration:float -> ?normalize:(string -> string) list -> @@ -17,10 +23,11 @@ ?tracking_url:string -> string -> (unit -> unit Promise.t) -> - t

Create a test to appear in a test suite.

  • category: the nested category to assign to the test. The category can be nested further using categorize or categorize_suites.
  • checked_output: determines how to capture the test's output. Defaults to no capture.
  • expected_outcome: whether a test is expected to complete without raising an exception (default) or by raising an exception.
  • max_duration: a time limit to run the test, in seconds. It is honored only in tests running in workers i.e. not with the -j0 option of the test program.
  • normalize: a list of functions applied in turn to transform the captured output before comparing it to the reference snapshot. See mask_line and other functions with the mask prefix which are provided for this purpose.
  • skipped: specify that the test must be skipped. This is intended for tests that give inconsistent results and need fixing. The string should explain why the test is being skipped. See also expected_outcome.
  • solo: specify that the test may not run in concurrently with other tests. The string should explain why.
  • tags: a list of tags to apply to the test. See Tag.
  • tolerate_chdir: by default, a test will fail if it modifies the current directory and doesn't restore it. This flag cancels this check. Note that Testo will always restore the current directory after running a test regardless of this setting.
val update : + t

Create a test to appear in a test suite.

  • category: the nested category to assign to the test. The category can be nested further using categorize or categorize_suites.
  • checked_output: determines how to capture the test's output. Defaults to no capture.
  • checked_output_files: specifies the test's output files that should remain the same from one test run to another.
  • expected_outcome: whether a test is expected to complete without raising an exception (default) or by raising an exception.
  • max_duration: a time limit to run the test, in seconds. It is honored only in tests running in workers i.e. not with the -j0 option of the test program.
  • normalize: a list of functions applied in turn to transform the captured output before comparing it to the reference snapshot. See mask_line and other functions with the mask prefix which are provided for this purpose.
  • skipped: specify that the test must be skipped. This is intended for tests that give inconsistent results and need fixing. The string should explain why the test is being skipped. See also expected_outcome.
  • solo: specify that the test may not run in concurrently with other tests. The string should explain why.
  • tags: a list of tags to apply to the test. See Tag.
  • tolerate_chdir: by default, a test will fail if it modifies the current directory and doesn't restore it. This flag cancels this check. Note that Testo will always restore the current directory after running a test regardless of this setting.
val update : ?broken:string option -> ?category:string list -> ?checked_output:checked_output_kind -> + ?checked_output_files:checked_output_file list -> ?expected_outcome:expected_outcome -> ?func:(unit -> unit Promise.t) -> ?max_duration:float option -> @@ -32,7 +39,7 @@ ?tolerate_chdir:bool -> ?tracking_url:string option -> t -> - t

Update some of the test's fields. This ensures that the test's unique identifier id is recomputed correctly. When specified, an optional property will replace the previous value.

Assertions and exceptions

Signaling a test failure is done by raising an exception. You may raise any exception to signal a test failure.

At this time, Testo doesn't provide advanced functions for checking a result against an expected value and printing these values nicely. For these, you may want to use `Alcotest.check` from the alcotest library.

exception Test_failure of string

The exception raised by fail

val fail : string -> unit

Raise the Test_failure exception with a message indicating the reason for the failure.

Temporary files and output redirection

val write_file : Fpath.t -> string -> unit

Write data to a regular file. Create the file if it doesn't exist. Erase any existing data.

Usage: write_file path data

val read_file : Fpath.t -> string

Read the contents of a regular file.

val with_temp_file : + t

Update some of the test's fields. This ensures that the test's unique identifier id is recomputed correctly. When specified, an optional property will replace the previous value.

Assertions and exceptions

Signaling a test failure is done by raising an exception. You may raise any exception to signal a test failure.

At this time, Testo doesn't provide advanced functions for checking a result against an expected value and printing these values nicely. For these, you may want to use `Alcotest.check` from the alcotest library.

exception Test_failure of string

The exception raised by fail

val fail : string -> unit

Raise the Test_failure exception with a message indicating the reason for the failure.

Temporary files and output redirection

val write_file : Fpath.t -> string -> unit

Write data to a regular file. Create the file if it doesn't exist. Erase any existing data.

Usage: write_file path data

val read_file : Fpath.t -> string

Read the contents of a regular file or symbolic link to a regular file.

val map_file : (string -> string) -> Fpath.t -> Fpath.t -> unit

map_file func src dst reads the contents of file (regular or symlink) src, applies func to its contents, and writes the result into file dst. If file dst already exists, it is truncated and overwritten. Otherwise, a regular file is created. If src and dst represent the same file, src will be overwritten with the new contents.

val copy_file : Fpath.t -> Fpath.t -> unit

Copy a file. copy_file src dst is a shortcut for map_file (fun data -> data) src dst.

val with_temp_file : ?contents:string -> ?persist:bool -> ?prefix:string -> @@ -45,7 +52,14 @@ ('a * string) Promise.t

with_capture stdout func evaluates func () while capturing the output of the given channel stdout as a string.

Environment control

val with_environment_variables : (string * string) list -> (unit -> 'a Promise.t) -> - 'a Promise.t

with_environment_variables ["FOO", "42"; "BAR", "hello"] func sets the environment variables FOO and BAR during the execution of func and then restores them to their original values.

Additionally, a test failure is produced if func modifies these environment variables without restoring them to the state in which it found them.

Due to a limitation in OCaml's "Unix" library, environment variables cannot be unset. If an environment variable was originally unset, restoring this original state isn't possible. Instead, the environment variable will be set to the empty string when with_environment_variables returns.

val with_chdir : Fpath.t -> (unit -> 'a) -> 'a

with_chdir path func changes the current directory associated with the process to path before calling the function func. The original value of the current directory is restored when the function terminates.

Output masking functions

Functions with the mask_ prefix are string replacement utilities to be used for masking the variable parts of test output in order to make them stable and comparable. This is for the normalize option of create.

Testo will keep a copy of the original, unmasked output for the developer to consult. In particular, this masking functionality will not prevent sensitive data such as passwords or secret keys from being stored in the local file system.

val mask_line : + 'a Promise.t

with_environment_variables ["FOO", "42"; "BAR", "hello"] func sets the environment variables FOO and BAR during the execution of func and then restores them to their original values.

Additionally, a test failure is produced if func modifies these environment variables without restoring them to the state in which it found them.

Due to a limitation in OCaml's "Unix" library, environment variables cannot be unset. If an environment variable was originally unset, restoring this original state isn't possible. Instead, the environment variable will be set to the empty string when with_environment_variables returns.

val with_chdir : Fpath.t -> (unit -> 'a) -> 'a

with_chdir path func changes the current directory associated with the process to path before calling the function func. The original value of the current directory is restored when the function returns or raises an exception.

val with_temp_dir : + ?chdir:bool -> + ?parent:Fpath.t -> + ?perms:int -> + ?prefix:string -> + ?suffix:string -> + (Fpath.t -> 'a) -> + 'a

with_temp_dir func creates temporary folder dir, calls the function func dir and returns its result. dir and its contents are removed before with_temp_dir returns or raises an exception. If chdir is set to true, the current folder is set to dir during the execution of func (default: false). parent is the parent folder of the temporary folder to create and defaults to Filename.get_temp_dir_name (). perms is the Unix-style permission mask used when creating dir with Sys.mkdir or equivalent. prefix and suffix are a prefix and a suffix to use for the name of dir (defaults: unspecified and subject to change).

val get_current_test : unit -> t option

Return the test currently running.

Output masking functions

Functions with the mask_ prefix are string replacement utilities to be used for masking the variable parts of test output in order to make them stable and comparable. This is for the normalize option of create.

Testo will keep a copy of the original, unmasked output for the developer to consult. In particular, this masking functionality will not prevent sensitive data such as passwords or secret keys from being stored in the local file system.

val mask_line : ?mask:string -> ?after:string -> ?before:string -> diff --git a/docs/reference/testo-util/Testo_util/index.html b/docs/reference/testo-util/Testo_util/index.html index 12b2206..577f918 100644 --- a/docs/reference/testo-util/Testo_util/index.html +++ b/docs/reference/testo-util/Testo_util/index.html @@ -1,2 +1,2 @@ -Testo_util (testo-util.Testo_util)

Module Testo_util

module CPU : sig ... end
module Debug : sig ... end
module Diff : sig ... end
module Error : sig ... end
module Filename_ : sig ... end
module Fpath_ : sig ... end
module Helpers : sig ... end
module Msg_from_master : sig ... end
module Msg_from_worker : sig ... end
module Multiprocess : sig ... end
module Slice : sig ... end
module Style : sig ... end
module Tag : sig ... end
+Testo_util (testo-util.Testo_util)

Module Testo_util

module CPU : sig ... end
module Debug : sig ... end
module Diff : sig ... end
module Error : sig ... end
module Filename_ : sig ... end
module Fpath_ : sig ... end
module Helpers : sig ... end
module Msg_from_master : sig ... end
module Msg_from_worker : sig ... end
module Multiprocess : sig ... end
module Slice : sig ... end
module Style : sig ... end
module Tag : sig ... end
diff --git a/docs/reference/testo/Testo/index.html b/docs/reference/testo/Testo/index.html index 30442ec..1ab8786 100644 --- a/docs/reference/testo/Testo/index.html +++ b/docs/reference/testo/Testo/index.html @@ -1,12 +1,18 @@ -Testo (testo.Testo)

Module Testo

Testo library - Utilities for writing OCaml test suites

Internal types

These types are documented in the library's source code in Types.ml. They are subject to frequent and unannounced changes at the whim of the library's authors. A casual user should not need them.

type expected_outcome =
  1. | Should_succeed
  2. | Should_fail of string
    (*

    explains why we expect this test to fail

    *)
type completion_status =
  1. | Test_function_returned
  2. | Test_function_raised_an_exception
  3. | Test_timeout
type fail_reason =
  1. | Raised_exception
  2. | Incorrect_output
  3. | Timeout
type outcome =
  1. | Succeeded
  2. | Failed of fail_reason
type captured_output =
  1. | Ignored of string
    (*

    unchecked combined output

    *)
  2. | Captured_stdout of string * string
    (*

    stdout, unchecked output

    *)
  3. | Captured_stderr of string * string
    (*

    stderr, unchecked output

    *)
  4. | Captured_stdout_stderr of string * string
    (*

    stdout, stderr

    *)
  5. | Captured_merged of string
    (*

    combined output

    *)
type expected_output =
  1. | Ignored
  2. | Expected_stdout of string
  3. | Expected_stderr of string
  4. | Expected_stdout_stderr of string * string
    (*

    stdout, stderr

    *)
  5. | Expected_merged of string
    (*

    combined output

    *)
type result = {
  1. completion_status : completion_status;
  2. captured_output : captured_output;
}
type missing_files =
  1. | Missing_files of Fpath.t list
type expectation = {
  1. expected_outcome : expected_outcome;
  2. expected_output : (expected_output, missing_files) Stdlib.Result.t;
}
type status = {
  1. expectation : expectation;
  2. result : (result, missing_files) Stdlib.Result.t;
}
type passing_status =
  1. | PASS
  2. | FAIL of fail_reason
  3. | XFAIL of fail_reason
  4. | XPASS
  5. | MISS of missing_files
type status_summary = {
  1. passing_status : passing_status;
  2. outcome : outcome;
  3. has_expected_output : bool;
}

Main interface

Test creation

type checked_output_kind

This type specifies what part of the output of a test (stdout, stderr) should be captured and compared against expectations.

Use the provided functions stdout, stderr, stdxxx, and split_stdout_stderr to create such an object.

val stdout : ?expected_stdout_path:Fpath.t -> unit -> checked_output_kind

Create an object of type checked_output_kind specifying that the test's standard output must be checked against a reference file.

val stderr : ?expected_stderr_path:Fpath.t -> unit -> checked_output_kind

Same as stdout but for capturing stderr instead.

val stdxxx : ?expected_stdxxx_path:Fpath.t -> unit -> checked_output_kind

Same as stdout but for capturing the combined stdout and stderr outputs.

val split_stdout_stderr : +Testo (testo.Testo)

Module Testo

Testo library - Utilities for writing OCaml test suites

Internal types

These types are documented in the library's source code in Types.ml. They are subject to frequent and unannounced changes at the whim of the library's authors. A casual user should not need them.

type expected_outcome =
  1. | Should_succeed
  2. | Should_fail of string
    (*

    explains why we expect this test to fail

    *)
type completion_status =
  1. | Test_function_returned
  2. | Test_function_raised_an_exception
  3. | Test_timeout
type fail_reason =
  1. | Raised_exception
  2. | Missing_output_file
  3. | Incorrect_output
  4. | Timeout
type outcome =
  1. | Succeeded
  2. | Failed of fail_reason
type checked_output_file

A test's output file whose contents captured from stdout or stderr should be checked and reported when it changes.

type checked_output_file_with_contents

A test's output file that is produced by the test function, checked, and reported when it changes.

type captured_output =
  1. | Ignored of string
    (*

    unchecked combined output

    *)
  2. | Captured_stdout of string * string
    (*

    stdout, unchecked output

    *)
  3. | Captured_stderr of string * string
    (*

    stderr, unchecked output

    *)
  4. | Captured_stdout_stderr of string * string
    (*

    stdout, stderr

    *)
  5. | Captured_merged of string
    (*

    combined output

    *)
type expected_output =
  1. | Ignored
  2. | Expected_stdout of string
  3. | Expected_stderr of string
  4. | Expected_stdout_stderr of string * string
    (*

    stdout, stderr

    *)
  5. | Expected_merged of string
    (*

    combined output

    *)
type result = {
  1. completion_status : completion_status;
  2. captured_output : captured_output;
  3. captured_output_files : checked_output_file_with_contents list;
  4. missing_output_files : Fpath.t list;
}
type missing_files =
  1. | Missing_files of Fpath.t list
type expectation = {
  1. expected_outcome : expected_outcome;
  2. expected_output : (expected_output, missing_files) Stdlib.Result.t;
  3. expected_output_files : (checked_output_file_with_contents, Fpath.t) + Stdlib.Result.t + list;
}
type status = {
  1. expectation : expectation;
  2. result : (result, missing_files) Stdlib.Result.t;
}
type passing_status =
  1. | PASS
  2. | FAIL of fail_reason
  3. | XFAIL of fail_reason
  4. | XPASS
  5. | MISS of missing_files
type status_summary = {
  1. passing_status : passing_status;
  2. outcome : outcome;
  3. has_expected_output : bool;
}

Main interface

Test creation

type checked_output_kind

This type specifies what part of the output of a test (stdout, stderr) should be captured and compared against expectations.

Use the provided functions stdout, stderr, stdxxx, and split_stdout_stderr to create such an object.

val stdout : ?expected_stdout_path:Fpath.t -> unit -> checked_output_kind

Create an object of type checked_output_kind specifying that the test's standard output must be checked against a reference file.

val stderr : ?expected_stderr_path:Fpath.t -> unit -> checked_output_kind

Same as stdout but for capturing stderr instead.

val stdxxx : ?expected_stdxxx_path:Fpath.t -> unit -> checked_output_kind

Same as stdout but for capturing the combined stdout and stderr outputs.

val split_stdout_stderr : ?expected_stdout_path:Fpath.t -> ?expected_stderr_path:Fpath.t -> unit -> - checked_output_kind

Same as stdxxx but keep stdout and stderr separate.

module Promise : module type of Promise

Wrapper allowing for asynchronous test functions (Lwt and such).

module Tag : module type of Testo_util.Tag

The type of tags which can be used to define subsets of tests precisely.

type t = private {
  1. id : string;
    (*

    Hash of the full name of the test, computed automatically.

    *)
  2. internal_full_name : string;
    (*

    Full name of the test, derived automatically from category and name.

    *)
  3. category : string list;
    (*

    Categories are made for organizing tests as a tree which is useful for display and filtering. A new category is created typically when grouping multiple test suites into one with 'categorize_suites' or when assigning a category to a list of tests with 'categorize'. e.g. ["food"; "fruit"; "kiwi"]

    *)
  4. name : string;
  5. func : unit -> unit Promise.t;
  6. broken : string option;
    (*

    If not None, the broken property causes the test to run normally but it will be ignored when determining the success of the test suite. This allows flaky tests to be kept around until they can be fixed. Use the string argument to explain briefly why the test is marked as broken. The --strict command-line option causes the broken status to be ignored i.e. a test run will fail if a broken test fails.

    *)
  7. checked_output : checked_output_kind;
  8. expected_outcome : expected_outcome;
  9. max_duration : float option;
    (*

    A time limit for the test when running in a detached worker, in seconds. This setting is ignored when running tests sequentially in the master process such as with -j0 or in solo mode.

    *)
  10. normalize : (string -> string) list;
    (*

    An optional function to rewrite any output data so as to mask the variable parts.

    *)
  11. skipped : string option;
    (*

    If not None, the skipped property causes a test to be skipped by Alcotest but still shown as "[SKIP]" rather than being omitted. The string should give a reason why the test is being skipped.

    *)
  12. solo : string option;
    (*

    If not None, this test will never run concurrently with other tests. The string should give a reason why the test should not run in parallel with other tests.

    *)
  13. tags : Tag.t list;
    (*

    Tags must be declared once using create_tag.

    *)
  14. tolerate_chdir : bool;
    (*

    If the test function changes the current directory without restoring it, it's an error unless this flag is set. All the tests in a test suite should share this field. To change the current directory temporarily and safely, use with_chdir.

    *)
  15. tracking_url : string option;
    (*

    A link to the relevant entry in a bug tracking system.

    *)
}

t is the type of a test. A test suite is a flat list of tests.

A test is at a minimum a name and a test function that raises exceptions to signal test failure. It is created with create or other similar functions provided by this module.

There are two main recommended ways of writing the test function:

1. With assert false:

Each test may use assert false to indicate that the test doesn't pass. This is the simplest way of failing while also showing the location of the failure. When using assert false, you should generally take care of printing the expected value and actual value to make debugging easier later.

2. With Alcotest.(check ...):

This is a little nicer because the error messages print something like "Expecting 'foo', got 'bar'". However, this can make tests slightly more complicated to write. If the test already prints the expected value and the actual value as its output, it's just easier to fail with assert false.

In any case, Alcotest will capture the output (stdout, stderr) of each test and put it in its own file so we can consult it later. Don't hesitate to log a lot during the execution of the test.

type test_with_status = t * status * status_summary
type subcommand_result =
  1. | Run_result of test_with_status list
  2. | Status_result of test_with_status list
  3. | Approve_result

The return type of each subcommand. It allows custom code to do something with the test data e.g. export to the JUnit format via the optional handle_subcommand_result argument of interpret_argv.

val create : + checked_output_kind

Same as stdxxx but keep stdout and stderr separate.

val checked_output_file : + ?snapshot_path:Fpath.t -> + string -> + checked_output_file

checked_output_file name creates the specification for a checked output file identified by the name name. Popular names include "results.txt" and "results.json". name must be a nonempty sequence of ASCII letters, digits, underscores, dashes, or periods. Periods are not allowed in first or last position. It is used by Testo in messaging and in file names. The snapshot_path option specifies an alternate location for the snapshot file that serves as the expectation for future test runs.

val stash_output_file : Fpath.t -> string -> unit

stash_output_file src_path dst_name copies a checked output file of the current test identified by its name dst_name into Testo's status workspace.

This function must be called by the test function for each checked output file.

module Promise : module type of Promise

Wrapper allowing for asynchronous test functions (Lwt and such).

module Tag : module type of Testo_util.Tag

The type of tags which can be used to define subsets of tests precisely.

type t = private {
  1. id : string;
    (*

    Hash of the full name of the test, computed automatically.

    *)
  2. internal_full_name : string;
    (*

    Full name of the test, derived automatically from category and name.

    *)
  3. category : string list;
    (*

    Categories are made for organizing tests as a tree which is useful for display and filtering. A new category is created typically when grouping multiple test suites into one with 'categorize_suites' or when assigning a category to a list of tests with 'categorize'. e.g. ["food"; "fruit"; "kiwi"]

    *)
  4. name : string;
  5. func : unit -> unit Promise.t;
  6. broken : string option;
    (*

    If not None, the broken property causes the test to run normally but it will be ignored when determining the success of the test suite. This allows flaky tests to be kept around until they can be fixed. Use the string argument to explain briefly why the test is marked as broken. The --strict command-line option causes the broken status to be ignored i.e. a test run will fail if a broken test fails.

    *)
  7. checked_output : checked_output_kind;
  8. checked_output_files : checked_output_file list;
    (*

    Files created by the test function whose contents must match a reference snapshot. The test function must copy its checked output files into Testo's workspace using stash_output_files.

    *)
  9. expected_outcome : expected_outcome;
  10. max_duration : float option;
    (*

    A time limit for the test when running in a detached worker, in seconds. This setting is ignored when running tests sequentially in the master process such as with -j0 or in solo mode.

    *)
  11. normalize : (string -> string) list;
    (*

    An optional function to rewrite any output data so as to mask the variable parts. This normalization is only applied to captured stdout or stderr, not to output files. If you wish to normalize output files, the test function should handle it, possibly with the provided map_file function.

    *)
  12. skipped : string option;
    (*

    If not None, the skipped property causes a test to be skipped by Alcotest but still shown as "[SKIP]" rather than being omitted. The string should give a reason why the test is being skipped.

    *)
  13. solo : string option;
    (*

    If not None, this test will never run concurrently with other tests. The string should give a reason why the test should not run in parallel with other tests.

    *)
  14. tags : Tag.t list;
    (*

    Tags must be declared once using create_tag.

    *)
  15. tolerate_chdir : bool;
    (*

    If the test function changes the current directory without restoring it, it's an error unless this flag is set. All the tests in a test suite should share this field. To change the current directory temporarily and safely, use with_chdir.

    *)
  16. tracking_url : string option;
    (*

    A link to the relevant entry in a bug tracking system.

    *)
}

t is the type of a test. A test suite is a flat list of tests.

A test is at a minimum a name and a test function that raises exceptions to signal test failure. It is created with create or other similar functions provided by this module.

There are two main recommended ways of writing the test function:

1. With assert false:

Each test may use assert false to indicate that the test doesn't pass. This is the simplest way of failing while also showing the location of the failure. When using assert false, you should generally take care of printing the expected value and actual value to make debugging easier later.

2. With Alcotest.(check ...):

This is a little nicer because the error messages print something like "Expecting 'foo', got 'bar'". However, this can make tests slightly more complicated to write. If the test already prints the expected value and the actual value as its output, it's just easier to fail with assert false.

In any case, Alcotest will capture the output (stdout, stderr) of each test and put it in its own file so we can consult it later. Don't hesitate to log a lot during the execution of the test.

type test_with_status = t * status * status_summary
type subcommand_result =
  1. | Run_result of test_with_status list
  2. | Status_result of test_with_status list
  3. | Approve_result

The return type of each subcommand. It allows custom code to do something with the test data e.g. export to the JUnit format via the optional handle_subcommand_result argument of interpret_argv.

val create : ?broken:string -> ?category:string list -> ?checked_output:checked_output_kind -> + ?checked_output_files:checked_output_file list -> ?expected_outcome:expected_outcome -> ?max_duration:float -> ?normalize:(string -> string) list -> @@ -17,10 +23,11 @@ ?tracking_url:string -> string -> (unit -> unit Promise.t) -> - t

Create a test to appear in a test suite.

  • category: the nested category to assign to the test. The category can be nested further using categorize or categorize_suites.
  • checked_output: determines how to capture the test's output. Defaults to no capture.
  • expected_outcome: whether a test is expected to complete without raising an exception (default) or by raising an exception.
  • max_duration: a time limit to run the test, in seconds. It is honored only in tests running in workers i.e. not with the -j0 option of the test program.
  • normalize: a list of functions applied in turn to transform the captured output before comparing it to the reference snapshot. See mask_line and other functions with the mask prefix which are provided for this purpose.
  • skipped: specify that the test must be skipped. This is intended for tests that give inconsistent results and need fixing. The string should explain why the test is being skipped. See also expected_outcome.
  • solo: specify that the test may not run in concurrently with other tests. The string should explain why.
  • tags: a list of tags to apply to the test. See Tag.
  • tolerate_chdir: by default, a test will fail if it modifies the current directory and doesn't restore it. This flag cancels this check. Note that Testo will always restore the current directory after running a test regardless of this setting.
val update : + t

Create a test to appear in a test suite.

  • category: the nested category to assign to the test. The category can be nested further using categorize or categorize_suites.
  • checked_output: determines how to capture the test's output. Defaults to no capture.
  • checked_output_files: specifies the test's output files that should remain the same from one test run to another.
  • expected_outcome: whether a test is expected to complete without raising an exception (default) or by raising an exception.
  • max_duration: a time limit to run the test, in seconds. It is honored only in tests running in workers i.e. not with the -j0 option of the test program.
  • normalize: a list of functions applied in turn to transform the captured output before comparing it to the reference snapshot. See mask_line and other functions with the mask prefix which are provided for this purpose.
  • skipped: specify that the test must be skipped. This is intended for tests that give inconsistent results and need fixing. The string should explain why the test is being skipped. See also expected_outcome.
  • solo: specify that the test may not run in concurrently with other tests. The string should explain why.
  • tags: a list of tags to apply to the test. See Tag.
  • tolerate_chdir: by default, a test will fail if it modifies the current directory and doesn't restore it. This flag cancels this check. Note that Testo will always restore the current directory after running a test regardless of this setting.
val update : ?broken:string option -> ?category:string list -> ?checked_output:checked_output_kind -> + ?checked_output_files:checked_output_file list -> ?expected_outcome:expected_outcome -> ?func:(unit -> unit Promise.t) -> ?max_duration:float option -> @@ -32,7 +39,7 @@ ?tolerate_chdir:bool -> ?tracking_url:string option -> t -> - t

Update some of the test's fields. This ensures that the test's unique identifier id is recomputed correctly. When specified, an optional property will replace the previous value.

Assertions and exceptions

Signaling a test failure is done by raising an exception. You may raise any exception to signal a test failure.

At this time, Testo doesn't provide advanced functions for checking a result against an expected value and printing these values nicely. For these, you may want to use `Alcotest.check` from the alcotest library.

exception Test_failure of string

The exception raised by fail

val fail : string -> unit

Raise the Test_failure exception with a message indicating the reason for the failure.

Temporary files and output redirection

val write_file : Fpath.t -> string -> unit

Write data to a regular file. Create the file if it doesn't exist. Erase any existing data.

Usage: write_file path data

val read_file : Fpath.t -> string

Read the contents of a regular file.

val with_temp_file : + t

Update some of the test's fields. This ensures that the test's unique identifier id is recomputed correctly. When specified, an optional property will replace the previous value.

Assertions and exceptions

Signaling a test failure is done by raising an exception. You may raise any exception to signal a test failure.

At this time, Testo doesn't provide advanced functions for checking a result against an expected value and printing these values nicely. For these, you may want to use `Alcotest.check` from the alcotest library.

exception Test_failure of string

The exception raised by fail

val fail : string -> unit

Raise the Test_failure exception with a message indicating the reason for the failure.

Temporary files and output redirection

val write_file : Fpath.t -> string -> unit

Write data to a regular file. Create the file if it doesn't exist. Erase any existing data.

Usage: write_file path data

val read_file : Fpath.t -> string

Read the contents of a regular file or symbolic link to a regular file.

val map_file : (string -> string) -> Fpath.t -> Fpath.t -> unit

map_file func src dst reads the contents of file (regular or symlink) src, applies func to its contents, and writes the result into file dst. If file dst already exists, it is truncated and overwritten. Otherwise, a regular file is created. If src and dst represent the same file, src will be overwritten with the new contents.

val copy_file : Fpath.t -> Fpath.t -> unit

Copy a file. copy_file src dst is a shortcut for map_file (fun data -> data) src dst.

val with_temp_file : ?contents:string -> ?persist:bool -> ?prefix:string -> @@ -45,7 +52,14 @@ ('a * string) Promise.t

with_capture stdout func evaluates func () while capturing the output of the given channel stdout as a string.

Environment control

val with_environment_variables : (string * string) list -> (unit -> 'a Promise.t) -> - 'a Promise.t

with_environment_variables ["FOO", "42"; "BAR", "hello"] func sets the environment variables FOO and BAR during the execution of func and then restores them to their original values.

Additionally, a test failure is produced if func modifies these environment variables without restoring them to the state in which it found them.

Due to a limitation in OCaml's "Unix" library, environment variables cannot be unset. If an environment variable was originally unset, restoring this original state isn't possible. Instead, the environment variable will be set to the empty string when with_environment_variables returns.

val with_chdir : Fpath.t -> (unit -> 'a) -> 'a

with_chdir path func changes the current directory associated with the process to path before calling the function func. The original value of the current directory is restored when the function terminates.

Output masking functions

Functions with the mask_ prefix are string replacement utilities to be used for masking the variable parts of test output in order to make them stable and comparable. This is for the normalize option of create.

Testo will keep a copy of the original, unmasked output for the developer to consult. In particular, this masking functionality will not prevent sensitive data such as passwords or secret keys from being stored in the local file system.

val mask_line : + 'a Promise.t

with_environment_variables ["FOO", "42"; "BAR", "hello"] func sets the environment variables FOO and BAR during the execution of func and then restores them to their original values.

Additionally, a test failure is produced if func modifies these environment variables without restoring them to the state in which it found them.

Due to a limitation in OCaml's "Unix" library, environment variables cannot be unset. If an environment variable was originally unset, restoring this original state isn't possible. Instead, the environment variable will be set to the empty string when with_environment_variables returns.

val with_chdir : Fpath.t -> (unit -> 'a) -> 'a

with_chdir path func changes the current directory associated with the process to path before calling the function func. The original value of the current directory is restored when the function returns or raises an exception.

val with_temp_dir : + ?chdir:bool -> + ?parent:Fpath.t -> + ?perms:int -> + ?prefix:string -> + ?suffix:string -> + (Fpath.t -> 'a) -> + 'a

with_temp_dir func creates temporary folder dir, calls the function func dir and returns its result. dir and its contents are removed before with_temp_dir returns or raises an exception. If chdir is set to true, the current folder is set to dir during the execution of func (default: false). parent is the parent folder of the temporary folder to create and defaults to Filename.get_temp_dir_name (). perms is the Unix-style permission mask used when creating dir with Sys.mkdir or equivalent. prefix and suffix are a prefix and a suffix to use for the name of dir (defaults: unspecified and subject to change).

val get_current_test : unit -> t option

Return the test currently running.

Output masking functions

Functions with the mask_ prefix are string replacement utilities to be used for masking the variable parts of test output in order to make them stable and comparable. This is for the normalize option of create.

Testo will keep a copy of the original, unmasked output for the developer to consult. In particular, this masking functionality will not prevent sensitive data such as passwords or secret keys from being stored in the local file system.

val mask_line : ?mask:string -> ?after:string -> ?before:string ->