From 1c25edb7d4888748b57c1dd5a359165f5c5a1a74 Mon Sep 17 00:00:00 2001 From: Anthony Leong Date: Sat, 27 Sep 2025 21:28:49 -0700 Subject: [PATCH 1/2] add job scheduling specs Signed-off-by: Anthony Leong --- spec/namespaces/search_relevance.yaml | 307 +++++++----------- spec/schemas/search_relevance._common.yaml | 199 +++++++++++- .../scheduled_experiments.yaml | 194 +++++++++++ 3 files changed, 505 insertions(+), 195 deletions(-) create mode 100644 tests/plugins/search_relevance/scheduled_experiments.yaml diff --git a/spec/namespaces/search_relevance.yaml b/spec/namespaces/search_relevance.yaml index c10f1e55d..6523ece1f 100644 --- a/spec/namespaces/search_relevance.yaml +++ b/spec/namespaces/search_relevance.yaml @@ -165,11 +165,6 @@ paths: responses: '200': $ref: '#/components/responses/search_relevance.get_experiments@200' - # content: - # application/json: - # schema: - # type: object - # $ref: '../schemas/search_relevance._common.yaml#/components/schemas/Experiment' delete: operationId: search_relevance.delete_experiments.0 x-operation-group: search_relevance.delete_experiments @@ -244,8 +239,46 @@ paths: responses: '200': $ref: '#/components/responses/search_relevance.get_stats@200' - - + /_plugins/_search_relevance/experiments/schedule: + get: + operationId: search_relevance.get_scheduled_experiments.0 + x-operation-group: search_relevance.get_scheduled_experiments + x-version-added: '3.1' + description: Gets the scheduled experiments. + responses: + '200': + $ref: '#/components/responses/search_relevance.get_scheduled_experiments@200' + post: + operationId: search_relevance.post_scheduled_experiments.0 + x-operation-group: search_relevance.post_scheduled_experiments + x-version-added: '3.1' + description: Creates a scheduled experiment. + requestBody: + $ref: '#/components/requestBodies/search_relevance.post_scheduled_experiments' + responses: + '200': + $ref: '#/components/responses/search_relevance.post_scheduled_experiments@200' + /_plugins/_search_relevance/experiments/schedule/{experiment_id}: + get: + operationId: search_relevance.get_scheduled_experiments.1 + x-operation-group: search_relevance.get_scheduled_experiments + x-version-added: '3.1' + description: Gets the scheduled experiments. + parameters: + - $ref: '#/components/parameters/search_relevance.get_scheduled_experiments::path.experiment_id' + responses: + '200': + $ref: '#/components/responses/search_relevance.get_scheduled_experiments@200' + delete: + operationId: search_relevance.delete_scheduled_experiments.0 + x-operation-group: search_relevance.delete_scheduled_experiments + x-version-added: '3.1' + description: Deletes a specified scheduled experiment. + parameters: + - $ref: '#/components/parameters/search_relevance.delete_scheduled_experiments::path.experiment_id' + responses: + '200': + $ref: '#/components/responses/search_relevance.delete_scheduled_experiments@200' components: parameters: search_relevance.get_query_sets::path.query_set_id: @@ -310,6 +343,22 @@ components: schema: type: string + search_relevance.get_scheduled_experiments::path.experiment_id: + name: experiment_id + in: path + required: true + description: The experiment id + schema: + type: + string + search_relevance.delete_scheduled_experiments::path.experiment_id: + name: experiment_id + in: path + required: true + description: The experiment id + schema: + type: + string search_relevance.get_node_stats::path.node_id: name: node_id in: path @@ -419,255 +468,151 @@ components: content: application/json: schema: - type: object - properties: - name: - type: string - description: - type: string - sampling: - type: string - querySetSize: - type: integer + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PostQuerySetsRequest' search_relevance.put_query_sets: content: application/json: schema: - type: object - properties: - name: - type: string - description: - type: string - sampling: - type: string - querySetQueries: - type: array - items: - type: object - # TODO: Replace the object type with the actual querySet - # schema. + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutQuerySetsRequest' search_relevance.put_judgments: content: application/json: schema: oneOf: - - type: object - description: The schema for a LLM judgment - properties: - name: - type: string - description: - type: string - type: - type: string - modelId: - type: string - querySetId: - type: string - searchConfigurationList: - type: array - items: - type: string - size: - type: integer - ignoreFailure: - type: boolean - contextFields: - type: array - items: - type: string - - type: object - description: The schema for a UBI judgment - properties: - name: - type: string - description: - type: string - type: - type: string - clickModel: - type: string - maxRank: - type: integer - - type: object - description: The schema for an import judgment - properties: - name: - type: string - description: - type: string - type: - type: string - judgmentRatings: - type: array - items: - type: object + - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutLLMJudgmentsRequest' + - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutUBIJudgmentsRequest' + - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutImportJudgmentsRequest' search_relevance.put_search_configurations: content: application/json: schema: - type: object - properties: - name: - type: string - index: - type: string - query: - type: string - searchPipeline: - type: string - + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutSearchConfigurationRequest' search_relevance.put_experiments: content: application/json: schema: anyOf: - - type: object - description: The schema for a pointwise and hybrid optimizer experiment - properties: - querySetId: - type: string - searchConfigurationList: - type: array - items: - type: string - judgmentList: - type: array - items: - type: string - size: - type: integer - type: - type: string - - type: object - description: The schema for a pairwise experiment - properties: - querySetId: - type: string - searchConfigurationList: - type: array - items: - type: string - size: - type: integer - type: - type: string + - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutHybridOptimizerExperimentRequest' + - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutPointwiseExperimentRequest' + - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutPairwiseExperimentRequest' + search_relevance.post_scheduled_experiments: + content: + application/json: + schema: + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PostScheduledExperimentsRequest' responses: search_relevance.get_query_sets@200: content: application/json: schema: - oneOf: - - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/QuerySet' - # TODO: Right Now, the generic object is the response of a get, but this can - # be more fleshed out to accomodate the specific return values of different gets. - - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/GenericObject' + $ref: '../schemas/_core.search.yaml#/components/schemas/SearchResult' search_relevance.post_query_sets@200: content: application/json: schema: - type: object - properties: - query_set_id: - type: string - query_set_result: - type: string + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PostQuerySetsResponse' search_relevance.put_query_sets@200: content: application/json: schema: - type: object - properties: - query_set_id: - type: string - query_set_result: - type: string + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutQuerySetsResponse' search_relevance.delete_query_sets@200: content: application/json: schema: - type: object - # TODO: The return value of deletes can be more specific. + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' search_relevance.get_judgments@200: content: application/json: schema: - oneOf: - - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/Judgment' - # TODO: Right Now, the generic object is the response of a get, but this can - # be more fleshed out to accomodate the specific return values of different gets. - - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/GenericObject' + $ref: '../schemas/_core.search.yaml#/components/schemas/SearchResult' search_relevance.put_judgments@200: content: application/json: - schema: - type: - object - properties: - judgment_id: - type: string + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutJudgmentsResponse' search_relevance.delete_judgments@200: content: application/json: schema: - type: object - # TODO: The return value of deletes can be more specific. + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' search_relevance.get_search_configurations@200: content: application/json: schema: - oneOf: - - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/SearchConfiguration' - # TODO: Right Now, the generic object is the response of a get, but this can - # be more fleshed out to accomodate the specific return values of different gets. - - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/GenericObject' + $ref: '../schemas/_core.search.yaml.yaml#/components/schemas/SearchResult' search_relevance.put_search_configurations@200: content: application/json: schema: - type: object - properties: - search_configuration_id: - type: string - search_configuration_result: - type: string + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutSearchConfigurationResponse' search_relevance.delete_search_configurations@200: content: application/json: schema: - type: object - # TODO: The return value of deletes can be more specific. + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' search_relevance.get_experiments@200: content: application/json: schema: - oneOf: - - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/Experiment' - # TODO: Right Now, the generic object is the response of a get, but this can - # be more fleshed out to accomodate the specific return values of different gets. - - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/GenericObject' + $ref: '../schemas/_core.search.yaml#/components/schemas/SearchResult' search_relevance.put_experiments@200: content: application/json: schema: - type: object - # TODO: The return value of puts can be more specific. + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutExperimentResponse' search_relevance.delete_experiments@200: content: application/json: schema: - type: object - # TODO: The return value of deletes can be more specific. + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' search_relevance.get_node_stats@200: content: application/json: schema: - type: object - # TODO: The return value of get node stats can be more specific. + allOf: + - $ref: '../schemas/nodes._common.yaml#/components/schemas/NodesResponseBase' + - type: object + properties: + cluster_name: + $ref: '../schemas/_common.yaml#/components/schemas/Name' + info: + type: object + all_nodes: + type: object + nodes: + type: object + required: + - nodes search_relevance.get_stats@200: content: application/json: schema: - type: object - # TODO: The return value of get stats should be more specific. \ No newline at end of file + allOf: + - $ref: '../schemas/nodes._common.yaml#/components/schemas/NodesResponseBase' + - type: object + properties: + cluster_name: + $ref: '../schemas/_common.yaml#/components/schemas/Name' + info: + type: object + all_nodes: + type: object + nodes: + type: object + required: + - nodes + search_relevance.get_scheduled_experiments@200: + content: + application/json: + schema: + $ref: '../schemas/_core.search.yaml#/components/schemas/SearchResult' + search_relevance.post_scheduled_experiments@200: + content: + application/json: + schema: + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PostScheduledExperimentsResponse' + search_relevance.delete_scheduled_experiments@200: + content: + application/json: + schema: + $ref: '../schemas/_common.yaml#/components/schemas/WriteResponseBase' \ No newline at end of file diff --git a/spec/schemas/search_relevance._common.yaml b/spec/schemas/search_relevance._common.yaml index 45cbee248..33b3f4c3b 100644 --- a/spec/schemas/search_relevance._common.yaml +++ b/spec/schemas/search_relevance._common.yaml @@ -85,21 +85,192 @@ components: # TODO: How to test that the output # of the results is correct because it is object which is dependent # on the input to the query set creation prior. - GenericObject: + PutExperimentResponse: type: object properties: - took: + experiment_id: + type: string + experiment_result: + type: string + PutSearchConfigurationResponse: + type: object + properties: + search_configuration_id: + type: string + search_configuration_result: + type: string + PutJudgmentsResponse: + type: object + properties: + judgment_id: + type: string + PostQuerySetsResponse: + type: object + properties: + query_set_id: + type: string + query_set_result: + type: string + PutQuerySetsResponse: + type: object + properties: + query_set_id: + type: string + query_set_result: + type: string + PutPairwiseExperimentRequest: + type: object + description: The schema for a pairwise experiment + properties: + querySetId: + type: string + searchConfigurationList: + type: array + items: + type: string + size: + type: integer + type: + type: string + PutHybridOptimizerExperimentRequest: + type: object + description: The schema for a hybrid optimizer experiment + properties: + querySetId: + type: string + searchConfigurationList: + type: array + items: + type: string + judgmentList: + type: array + items: + type: string + size: + type: integer + type: + type: string + PutPointwiseExperimentRequest: + type: object + description: The schema for a pointwise experiment + properties: + querySetId: + type: string + searchConfigurationList: + type: array + items: + type: string + judgmentList: + type: array + items: + type: string + size: + type: integer + type: + type: string + PostQuerySetsRequest: + type: object + description: The schema for creating a query set + properties: + name: + type: string + description: + type: string + sampling: + type: string + querySetSize: + type: integer + PutQuerySetsRequest: + type: object + description: The schema for updating a query set + properties: + name: + type: string + description: + type: string + sampling: + type: string + querySetQueries: + type: array + items: + type: object + PutLLMJudgmentsRequest: + type: object + description: The schema for a LLM judgment + properties: + name: + type: string + description: + type: string + type: + type: string + modelId: + type: string + querySetId: + type: string + searchConfigurationList: + type: array + items: + type: string + size: type: integer - timed_out: + ignoreFailure: type: boolean - _shards: - type: object - # TODO: How does one test that the output - # of the _shards is correct because it is object which is dependent - # on the input to the query set creation prior. - hits: - type: object - # TODO: How to test that the output - # of the hits is correct because it is object which is dependent - # on the input to the query set creation prior. The items within - # hits also contain the result objects of the different apis. + contextFields: + type: array + items: + type: string + PutUBIJudgmentsRequest: + type: object + description: The schema for a UBI judgment + properties: + name: + type: string + description: + type: string + type: + type: string + clickModel: + type: string + maxRank: + type: integer + PutImportJudgmentsRequest: + type: object + description: The schema for an import judgment + properties: + name: + type: string + description: + type: string + type: + type: string + judgmentRatings: + type: array + items: + type: object + PutSearchConfigurationRequest: + type: object + properties: + name: + type: string + index: + type: string + query: + type: string + searchPipeline: + type: string + PostScheduledExperimentsRequest: + type: object + description: The schema for scheduling experiments + properties: + experimentId: + type: string + cronExpression: + type: string + PostScheduledExperimentsResponse: + type: object + properties: + job_id: + type: string + job_result: + type: string diff --git a/tests/plugins/search_relevance/scheduled_experiments.yaml b/tests/plugins/search_relevance/scheduled_experiments.yaml new file mode 100644 index 000000000..38764091a --- /dev/null +++ b/tests/plugins/search_relevance/scheduled_experiments.yaml @@ -0,0 +1,194 @@ +$schema: ../../../json_schemas/test_story.schema.yaml + +description: Tests the creation and deletion of experiments. + +prologues: + - path: /_cluster/settings + method: PUT + request: + payload: + transient: + plugins.search_relevance.workbench_enabled: true + response: + status: 201 + - path: /_cluster/settings + method: PUT + request: + payload: + transient: + plugins.search_relevance.scheduled_experiments_enabled: true + response: + status: 201 + - path: /{index}/_doc/{doc_id} + method: POST + parameters: + index: sample_index + doc_id: 1 + request: + payload: + name: banana + price: 1.99 + description: this is a banana + response: + status: 201 + - path: /{index}/_doc/{doc_id} + method: POST + parameters: + index: sample_index + doc_id: 2 + request: + payload: + name: apple + price: 3.99 + description: this is an apple + response: + status: 201 + - path: /{index}/_doc/{doc_id} + method: POST + parameters: + index: sample_index + doc_id: 3 + request: + payload: + name: test + price: 19.99 + description: this is a test + response: + status: 201 + - id: query_set_sampling + path: /_plugins/_search_relevance/query_sets + method: POST + request: + payload: + name: test03 + description: test03 + sampling: random + querySetSize: 200 + response: + status: 200 + output: # Save the query id for later use. + query_set_id: payload.query_set_id + - id: search_configuration1 + path: /_plugins/_search_relevance/search_configurations + method: PUT + request: + payload: + name: simple + query: "{\"query\": {\n\"match_all\": {}}}" + index: sample_index + searchPipeline: test_pipeline + response: + status: 200 + output: # Save the search configuration id for later use. + search_configuration_id: payload.search_configuration_id + - id: search_configuration2 + path: /_plugins/_search_relevance/search_configurations + method: PUT + request: + payload: + name: multimatch + query: '{"query":{"multi_match":{"query":"%SearchText%","fields":["id","title","category","bullets","description","attrs.Brand","attrs.Color"]}}}' + index: sample_index + searchPipeline: test_pipeline + response: + status: 200 + output: # Save the search configuration id for later use. + search_configuration_id: payload.search_configuration_id + - id: judgment1 + path: /_plugins/_search_relevance/judgments + method: PUT + request: + payload: + name: UBIbaseline + description: UBI judgment + type: UBI_JUDGMENT + clickModel: coec + maxRank: 10 + response: + status: 200 + output: # Save the judgment id for later use. + judgment_id: payload.judgment_id + - id: pointwise_experiment + path: /_plugins/_search_relevance/experiments + method: PUT + request: + payload: + querySetId: ${query_set_sampling.query_set_id} + searchConfigurationList: + - ${search_configuration1.search_configuration_id} + judgmentList: + - ${judgment1.judgment_id} + size: 10 + type: POINTWISE_EVALUATION + response: + status: 200 + output: # Save the search experiment_id id for later use. + experiment_id: payload.experiment_id + +epilogues: + - path: /_plugins/_search_relevance/judgments/{judgment_id} + method: DELETE + parameters: + judgment_id: ${judgment1.judgment_id} + response: + status: 200 + - path: /_plugins/_search_relevance/search_configurations/{search_configuration_id} + method: DELETE + parameters: + search_configuration_id: ${search_configuration1.search_configuration_id} + response: + status: 200 + - path: /_plugins/_search_relevance/search_configurations/{search_configuration_id} + method: DELETE + parameters: + search_configuration_id: ${search_configuration2.search_configuration_id} + response: + status: 200 + - path: /_plugins/_search_relevance/query_sets/{query_set_id} + method: DELETE + parameters: + query_set_id: ${query_set_sampling.query_set_id} + response: + status: 200 + - path: /_plugins/_search_relevance/experiments/{experiment_id} + method: DELETE + parameters: + experiment_id: ${pointwise_experiment.experiment_id} + response: + status: 200 + - path: /sample_index + method: DELETE + status: [200, 404] + +chapters: + - synopsis: Creating a scheduled experiment. + id: scheduled_experiment + path: /_plugins/_search_relevance/experiments/schedule + method: POST + request: + payload: + experimentId: ${pointwise_experiment.experiment_id} + cronExpression: "1 1 1 1 1" + response: + status: 200 + output: # Save the search experiment_id id for later use. + experiment_id: payload.job_id + - synopsis: List all the scheduled experiments created. + path: /_plugins/_search_relevance/experiments/schedule + method: GET + response: + status: 200 + - synopsis: Retrieve a specific scheduled experiment. + path: /_plugins/_search_relevance/experiments/schedule/{experiment_id} + method: GET + parameters: + experiment_id: ${scheduled_experiment.experiment_id} + response: + status: 200 + - synopsis: Delete the scheduled experiment. + path: /_plugins/_search_relevance/experiments/schedule/{experiment_id} + method: DELETE + parameters: + experiment_id: ${scheduled_experiment.experiment_id} + response: + status: 200 \ No newline at end of file From 80d6b4049da6cb3723d1bcc7bb5e5c75bacb140d Mon Sep 17 00:00:00 2001 From: Anthony Leong Date: Sat, 27 Sep 2025 21:39:48 -0700 Subject: [PATCH 2/2] styling fixes Signed-off-by: Anthony Leong --- spec/namespaces/search_relevance.yaml | 3 ++- .../search_relevance/scheduled_experiments.yaml | 10 +--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/spec/namespaces/search_relevance.yaml b/spec/namespaces/search_relevance.yaml index 6523ece1f..7e2222390 100644 --- a/spec/namespaces/search_relevance.yaml +++ b/spec/namespaces/search_relevance.yaml @@ -529,7 +529,8 @@ components: search_relevance.put_judgments@200: content: application/json: - $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutJudgmentsResponse' + schema: + $ref: '../schemas/search_relevance._common.yaml#/components/schemas/PutJudgmentsResponse' search_relevance.delete_judgments@200: content: application/json: diff --git a/tests/plugins/search_relevance/scheduled_experiments.yaml b/tests/plugins/search_relevance/scheduled_experiments.yaml index 38764091a..87249a2b9 100644 --- a/tests/plugins/search_relevance/scheduled_experiments.yaml +++ b/tests/plugins/search_relevance/scheduled_experiments.yaml @@ -11,14 +11,6 @@ prologues: plugins.search_relevance.workbench_enabled: true response: status: 201 - - path: /_cluster/settings - method: PUT - request: - payload: - transient: - plugins.search_relevance.scheduled_experiments_enabled: true - response: - status: 201 - path: /{index}/_doc/{doc_id} method: POST parameters: @@ -168,7 +160,7 @@ chapters: request: payload: experimentId: ${pointwise_experiment.experiment_id} - cronExpression: "1 1 1 1 1" + cronExpression: 1 1 1 1 1 response: status: 200 output: # Save the search experiment_id id for later use.