Skip to content

Commit 5b19aeb

Browse files
github-actions[bot]lerman25
github-actions[bot]
and
lerman25
authored
[8.0] [MOD-8204] int8 micro benchmarks (#590)
[MOD-8204] int8 micro benchmarks (#583) * gignore * int8 single * yml update * fix bm_files * batch iterator * newline at eof * remove multi from bm_files (not relevant yet) * fix extra / on bm_files * moved macro to cpp, change BF range * comment on INT8 range query macros * Minor details * Another minor detail * format * Update tests/benchmark/run_files/bm_basics_single_int8.cpp Co-authored-by: meiravgri <[email protected]> --------- Co-authored-by: meiravgri <[email protected]> (cherry picked from commit 3291f63) Co-authored-by: lerman25 <[email protected]>
1 parent 12517dc commit 5b19aeb

File tree

13 files changed

+220
-3
lines changed

13 files changed

+220
-3
lines changed

‎.github/workflows/benchmark.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ on:
1111
options:
1212
- benchmarks-all
1313
- benchmarks-default
14+
- bm-basics-int8-single
1415
- bm-basics-fp32-single
1516
- bm-basics-fp32-multi
1617
- bm-basics-fp64-single

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# Ignore benchmark fetched data but not the source file
1414
/tests/benchmark/data/*
1515
!/tests/benchmark/data/hnsw_indices
16-
!/tests/benchmark/data/serializer.py
16+
!/tests/benchmark/data/*.py
1717

1818
# Prerequisites
1919
*.d

‎src/python_bindings/bindings.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -219,6 +219,8 @@ class PyVecSimIndex {
219219
return rawVectorsAsNumpy<bfloat16, float, float>(label, dim);
220220
} else if (info.commonInfo.basicInfo.type == VecSimType_FLOAT16) {
221221
return rawVectorsAsNumpy<float16, float, float>(label, dim);
222+
} else if (info.commonInfo.basicInfo.type == VecSimType_INT8) {
223+
return rawVectorsAsNumpy<int8_t, float>(label, dim);
222224
} else {
223225
throw std::runtime_error("Invalid vector data type");
224226
}
@@ -439,6 +441,7 @@ class PyHNSWLibIndex : public PyVecSimIndex {
439441
return dynamic_cast<HNSWIndex<int8_t, float> *>(this->index.get())
440442
->checkIntegrity()
441443
.valid_state;
444+
442445
} else {
443446
throw std::runtime_error("Invalid index data type");
444447
}

‎tests/benchmark/benchmarks.sh‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,15 @@ if [ -z "$BM_TYPE" ] || [ "$BM_TYPE" = "benchmarks-all" ]; then
88
done
99
done
1010
done
11+
echo basics_single_int8
12+
echo batch_iterator_single_int8
1113
echo updated_index_single_fp32
1214
echo spaces_fp32
1315
echo spaces_fp64
1416
echo spaces_bf16
1517
echo spaces_fp16
16-
echo spaces_int8
18+
echo spaces_int8
19+
1720
elif [ "$BM_TYPE" = "benchmarks-default" ]; then
1821
echo basics_single_fp32
1922
echo basics_multi_fp32
@@ -39,7 +42,8 @@ elif [ "$BM_TYPE" = "bm-basics-fp16-single" ] ; then
3942
echo basics_single_fp16
4043
elif [ "$BM_TYPE" = "bm-basics-fp16-multi" ] ; then
4144
echo basics_multi_fp16
42-
45+
elif [ "$BM_TYPE" = "bm-basics-int8-single" ] ; then
46+
echo basics_single_int8
4347
# Batch iterator benchmarks
4448
elif [ "$BM_TYPE" = "bm-batch-iter-fp32-single" ] ; then
4549
echo batch_iterator_single_fp32
@@ -57,6 +61,8 @@ elif [ "$BM_TYPE" = "bm-batch-iter-fp16-single" ] ; then
5761
echo batch_iterator_single_fp16
5862
elif [ "$BM_TYPE" = "bm-batch-iter-fp16-multi" ] ; then
5963
echo batch_iterator_multi_fp16
64+
elif [ "$BM_TYPE" = "bm-batch-iter-int8-single" ] ; then
65+
echo batch_iterator_single_int8
6066

6167
# Updated index benchmarks
6268
elif [ "$BM_TYPE" = "bm-updated-fp32-single" ] ; then

‎tests/benchmark/bm_definitions.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ using fp32_index_t = IndexType<VecSimType_FLOAT32, float, float>;
1616
using fp64_index_t = IndexType<VecSimType_FLOAT64, double, double>;
1717
using bf16_index_t = IndexType<VecSimType_BFLOAT16, vecsim_types::bfloat16, float>;
1818
using fp16_index_t = IndexType<VecSimType_FLOAT16, vecsim_types::float16, float>;
19+
using int8_index_t = IndexType<VecSimType_INT8, int8_t, float>;
1920

2021
#define INDICES BM_VecSimIndex<index_type_t>::indices
2122
#define QUERIES BM_VecSimIndex<index_type_t>::queries

‎tests/benchmark/bm_files.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,10 @@ elif [ "$BM_TYPE" = "bm-basics-fp16-single" ] \
2626
|| [ "$BM_TYPE" = "bm-batch-iter-fp16-multi" ]
2727
then
2828
file_name="basic_fp16"
29+
elif [ "$BM_TYPE" = "bm-basics-int8-single" ] \
30+
|| [ "$BM_TYPE" = "bm-batch-iter-int8-single" ]
31+
then
32+
file_name="basic_int8"
2933
elif [ "$BM_TYPE" = "bm-updated-fp32-single" ]; then
3034
file_name="updated"
3135
else

‎tests/benchmark/bm_initialization/bm_basics_initialize_int8.h‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#pragma once
2+
/**************************************
3+
Define and register tests
4+
NOTE: benchmarks' tests order can affect their results. Please add new benchmarks at the end of
5+
the file.
6+
***************************************/
7+
8+
// Memory BF
9+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, FLAT), int8_index_t)
10+
(benchmark::State &st) { Memory_FLAT(st); }
11+
BENCHMARK_REGISTER_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, FLAT))->Iterations(1);
12+
13+
// Memory HNSW
14+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, HNSW), int8_index_t)
15+
(benchmark::State &st) { Memory_HNSW(st); }
16+
BENCHMARK_REGISTER_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, HNSW))->Iterations(1);
17+
18+
// Memory Tiered
19+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, Tiered), int8_index_t)
20+
(benchmark::State &st) { Memory_Tiered(st); }
21+
BENCHMARK_REGISTER_F(BM_VecSimCommon, BM_FUNC_NAME(Memory, Tiered))->Iterations(1);
22+
23+
// AddLabel
24+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_ADD_LABEL, int8_index_t)
25+
(benchmark::State &st) { AddLabel(st); }
26+
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_BF);
27+
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_HNSWLIB);
28+
29+
// DeleteLabel Registration. Definition is placed in the .cpp file.
30+
REGISTER_DeleteLabel(BM_FUNC_NAME(DeleteLabel, BF));
31+
REGISTER_DeleteLabel(BM_FUNC_NAME(DeleteLabel, HNSW));
32+
33+
// TopK BF
34+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(TopK, BF), int8_index_t)
35+
(benchmark::State &st) { TopK_BF(st); }
36+
REGISTER_TopK_BF(BM_VecSimCommon, BM_FUNC_NAME(TopK, BF));
37+
38+
// TopK HNSW
39+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(TopK, HNSW), int8_index_t)
40+
(benchmark::State &st) { TopK_HNSW(st); }
41+
REGISTER_TopK_HNSW(BM_VecSimCommon, BM_FUNC_NAME(TopK, HNSW));
42+
43+
// TopK Tiered HNSW
44+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimCommon, BM_FUNC_NAME(TopK, Tiered), int8_index_t)
45+
(benchmark::State &st) { TopK_Tiered(st); }
46+
REGISTER_TopK_Tiered(BM_VecSimCommon, BM_FUNC_NAME(TopK, Tiered));
47+
48+
// Range BF
49+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, BF), int8_index_t)
50+
(benchmark::State &st) { Range_BF(st); }
51+
REGISTER_Range_INT8_BF(BM_FUNC_NAME(Range, BF));
52+
53+
// Range HNSW
54+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_FUNC_NAME(Range, HNSW), int8_index_t)
55+
(benchmark::State &st) { Range_HNSW(st); }
56+
REGISTER_Range_INT8_HNSW(BM_FUNC_NAME(Range, HNSW));
57+
58+
// Tiered HNSW add/delete benchmarks
59+
REGISTER_AddLabel(BM_ADD_LABEL, VecSimAlgo_TIERED);
60+
REGISTER_DeleteLabel(BM_FUNC_NAME(DeleteLabel, Tiered));
61+
62+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_ADD_LABEL_ASYNC, int8_index_t)
63+
(benchmark::State &st) { AddLabel_AsyncIngest(st); }
64+
BENCHMARK_REGISTER_F(BM_VecSimBasics, BM_ADD_LABEL_ASYNC)
65+
->UNIT_AND_ITERATIONS->Arg(VecSimAlgo_TIERED)
66+
->ArgName("VecSimAlgo_TIERED");
67+
68+
BENCHMARK_TEMPLATE_DEFINE_F(BM_VecSimBasics, BM_DELETE_LABEL_ASYNC, int8_index_t)
69+
(benchmark::State &st) { DeleteLabel_AsyncRepair(st); }

‎tests/benchmark/bm_initialization/bm_batch_initialize_int8.h‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#pragma once
2+
3+
/**************************************
4+
Define and register benchmarks for batch iterator with index of data type int8
5+
NOTE: benchmarks' tests order can affect their results. Please add new benchmarks at the end of
6+
the file.
7+
***************************************/
8+
9+
// Fixed size batch BF
10+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(BF, FixedBatchSize), int8_index_t)
11+
(benchmark::State &st) { BF_FixedBatchSize(st); }
12+
REGISTER_FixedBatchSize(BM_FUNC_NAME(BF, FixedBatchSize));
13+
14+
// Variable size batch BF
15+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(BF, VariableBatchSize), int8_index_t)
16+
(benchmark::State &st) { BF_VariableBatchSize(st); }
17+
REGISTER_VariableBatchSize(BM_FUNC_NAME(BF, VariableBatchSize));
18+
19+
// Batches to hadoc BF
20+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(BF, BatchesToAdhocBF), int8_index_t)
21+
(benchmark::State &st) { BF_BatchesToAdhocBF(st); }
22+
REGISTER_BatchesToAdhocBF(BM_FUNC_NAME(BF, BatchesToAdhocBF));
23+
24+
// Fixed size batch HNSW
25+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(HNSW, FixedBatchSize), int8_index_t)
26+
(benchmark::State &st) { HNSW_FixedBatchSize(st); }
27+
REGISTER_FixedBatchSize(BM_FUNC_NAME(HNSW, FixedBatchSize));
28+
29+
// Variable size batch BF
30+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(HNSW, VariableBatchSize), int8_index_t)
31+
(benchmark::State &st) { HNSW_VariableBatchSize(st); }
32+
REGISTER_VariableBatchSize(BM_FUNC_NAME(HNSW, VariableBatchSize));
33+
34+
// Batches to hadoc HSNW
35+
BENCHMARK_TEMPLATE_DEFINE_F(BM_BatchIterator, BM_FUNC_NAME(HNSW, BatchesToAdhocBF), int8_index_t)
36+
(benchmark::State &st) { HNSW_BatchesToAdhocBF(st); }
37+
38+
REGISTER_HNSW_BatchesToAdhocBF(BM_FUNC_NAME(HNSW, BatchesToAdhocBF));

‎tests/benchmark/bm_vecsim_index.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ std::vector<std::vector<vecsim_types::bfloat16>> BM_VecSimIndex<bf16_index_t>::q
5151
template <>
5252
std::vector<std::vector<vecsim_types::float16>> BM_VecSimIndex<fp16_index_t>::queries{};
5353

54+
template <>
55+
std::vector<std::vector<int8_t>> BM_VecSimIndex<int8_index_t>::queries{};
56+
5457
template <>
5558
std::vector<VecSimIndex *> BM_VecSimIndex<fp32_index_t>::indices{};
5659

@@ -63,6 +66,9 @@ std::vector<VecSimIndex *> BM_VecSimIndex<bf16_index_t>::indices{};
6366
template <>
6467
std::vector<VecSimIndex *> BM_VecSimIndex<fp16_index_t>::indices{};
6568

69+
template <>
70+
std::vector<VecSimIndex *> BM_VecSimIndex<int8_index_t>::indices{};
71+
6672
template <typename index_type_t>
6773
BM_VecSimIndex<index_type_t>::~BM_VecSimIndex() {
6874
ref_count--;

‎tests/benchmark/data/hnsw_indices/hnsw_indices_all.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,3 +24,6 @@ https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/dbpedia-cosine-dim768-fp
2424

2525
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/fashion_images_multi_value-cosine-dim512-M64-efc512-fp16.hnsw_v3
2626
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/fashion_images_multi_value-cosine-dim512-fp16-test_vectors.raw
27+
28+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia-cosine-dim1024-M64-efc512-int8.hnsw_v3
29+
https://dev.cto.redis.s3.amazonaws.com/VectorSimilarity/wipedia-cosine-dim1024-int8-test_vectors.raw

0 commit comments

Comments
 (0)