Skip to content

Provide an explicit state for the flow classification process #2942

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Provide an explicit state for the flow classification process #2942

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 16 additions & 17 deletions example/ndpiReader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ void ndpiCheckHostStringMatch(char *testChar) {
detected_protocol.proto.master_protocol = 0;
detected_protocol.category = match.protocol_category;

ndpi_protocol2name(ndpi_str, detected_protocol, appBufStr,
ndpi_protocol2name(ndpi_str, detected_protocol.proto, appBufStr,
sizeof(appBufStr));

printf("Match Found for string [%s] -> P(%d) B(%d) C(%d) => %s %s %s\n",
Expand Down Expand Up @@ -559,7 +559,7 @@ static void ndpiCheckIPMatch(char *testChar) {
memset(&detected_protocol, 0, sizeof(ndpi_protocol));
detected_protocol.proto.app_protocol = ndpi_map_ndpi_id_to_user_proto_id(ndpi_str, ret);

ndpi_protocol2name(ndpi_str, detected_protocol, appBufStr,
ndpi_protocol2name(ndpi_str, detected_protocol.proto, appBufStr,
sizeof(appBufStr));

printf("Match Found for IP %s, port %d -> %s (%d)\n",
Expand Down Expand Up @@ -1776,11 +1776,11 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
);

fprintf(csv_fp, "%s|",
ndpi_protocol2id(flow->detected_protocol, buf, sizeof(buf)));
ndpi_protocol2id(flow->detected_protocol.proto, buf, sizeof(buf)));

fprintf(csv_fp, "%s|%s|%s|%s|",
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol, buf, sizeof(buf)),
flow->detected_protocol.proto, buf, sizeof(buf)),
ndpi_stack2str(ndpi_thread_info[thread_id].workflow->ndpi_struct,
&flow->detected_protocol.protocol_stack, buf2, sizeof(buf2)),
ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
Expand Down Expand Up @@ -1919,7 +1919,7 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
flow->detected_protocol.proto.master_protocol,
flow->detected_protocol.proto.app_protocol,
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol, buf1, sizeof(buf1))
flow->detected_protocol.proto, buf1, sizeof(buf1))
);
}
}
Expand Down Expand Up @@ -1961,14 +1961,14 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa

#ifdef NDPI_EXTENDED_SANITY_CHECKS
/* Be sure new stack logic is compatible with legacy code */
assert(ndpi_stack_get_upper_proto(&flow->detected_protocol.protocol_stack) == ndpi_get_upper_proto(flow->detected_protocol));
assert(ndpi_stack_get_lower_proto(&flow->detected_protocol.protocol_stack) == ndpi_get_lower_proto(flow->detected_protocol));
assert(ndpi_stack_get_upper_proto(&flow->detected_protocol.protocol_stack) == ndpi_get_upper_proto(flow->detected_protocol.proto));
assert(ndpi_stack_get_lower_proto(&flow->detected_protocol.protocol_stack) == ndpi_get_lower_proto(flow->detected_protocol.proto));
#endif

fprintf(out, "%s/%s][Stack: %s][IP: %u/%s]",
ndpi_protocol2id(flow->detected_protocol, buf, sizeof(buf)),
ndpi_protocol2id(flow->detected_protocol.proto, buf, sizeof(buf)),
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol, buf1, sizeof(buf1)),
flow->detected_protocol.proto, buf1, sizeof(buf1)),
ndpi_stack2str(ndpi_thread_info[thread_id].workflow->ndpi_struct,
&flow->detected_protocol.protocol_stack, buf2, sizeof(buf2)),
flow->detected_protocol.protocol_by_ip,
Expand Down Expand Up @@ -2002,7 +2002,7 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa

fprintf(out, "[%s]",
ndpi_is_encrypted_proto(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol) ? "Encrypted" : "ClearText");
flow->detected_protocol.proto) ? "Encrypted" : "ClearText");

fprintf(out, "[Confidence: %s]", ndpi_confidence_get_name(flow->confidence));

Expand Down Expand Up @@ -2567,14 +2567,13 @@ static void node_proto_guess_walker(const void *node, ndpi_VISIT which, int dept

if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */
if((!flow->detection_completed) && flow->ndpi_flow) {
u_int8_t proto_guessed;

malloc_size_stats = 1;
flow->detected_protocol = ndpi_detection_giveup(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->ndpi_flow, &proto_guessed);
flow->ndpi_flow);
malloc_size_stats = 0;

if(proto_guessed) ndpi_thread_info[thread_id].workflow->stats.guessed_flow_protocols++;
if(flow->ndpi_flow->protocol_was_guessed) ndpi_thread_info[thread_id].workflow->stats.guessed_flow_protocols++;
}

process_ndpi_collected_info(ndpi_thread_info[thread_id].workflow, flow);
Expand Down Expand Up @@ -2976,7 +2975,7 @@ static void port_stats_walker(const void *node, ndpi_VISIT which, int depth, voi
/* get app level protocol */
if(flow->detected_protocol.proto.master_protocol) {
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol, proto, sizeof(proto));
flow->detected_protocol.proto, proto, sizeof(proto));
} else {
strncpy(proto, ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->detected_protocol.proto.app_protocol),sizeof(proto) - 1);
Expand Down Expand Up @@ -3084,7 +3083,7 @@ static void dump_realtime_protocol(struct ndpi_workflow * workflow, struct ndpi_
snprintf(dstip, sizeof(dstip), "[%s]", flow->dst_name ? flow->dst_name : "");
}

ndpi_protocol2name(workflow->ndpi_struct, flow->detected_protocol, app_name, sizeof(app_name));
ndpi_protocol2name(workflow->ndpi_struct, flow->detected_protocol.proto, app_name, sizeof(app_name));

if (ret == 1) {
fprintf(out, "Detected Realtime protocol %s --> [%s] %s:%d <--> %s:%d app=%s <%s>\n",
Expand Down Expand Up @@ -3994,7 +3993,7 @@ static void printFlowsStats() {
fprintf(out, "\t%u\t%-10s\t%s:%u <-> %s:%u\t[",
i,
ndpi_protocol2name(ndpi_thread_info[0].workflow->ndpi_struct,
all_flows[i].flow->detected_protocol, buf, sizeof(buf)),
all_flows[i].flow->detected_protocol.proto, buf, sizeof(buf)),
all_flows[i].flow->src_name ? all_flows[i].flow->src_name : "",
ntohs(all_flows[i].flow->src_port),
all_flows[i].flow->dst_name ? all_flows[i].flow->dst_name : "",
Expand Down Expand Up @@ -4942,7 +4941,7 @@ static void ndpi_process_packet(u_char *args,
}
trailer->flow_risk_info[sizeof(trailer->flow_risk_info) - 1] = '\0';
trailer->proto.master_protocol = htons(p.proto.master_protocol), trailer->proto.app_protocol = htons(p.proto.app_protocol);
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p, trailer->name, sizeof(trailer->name));
ndpi_protocol2name(ndpi_thread_info[thread_id].workflow->ndpi_struct, p.proto, trailer->name, sizeof(trailer->name));

/* Metadata */
/* Metadata are (all) available in `flow` only after nDPI completed its work!
Expand Down
6 changes: 2 additions & 4 deletions example/ndpiSimpleIntegration.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -876,12 +876,10 @@ static void ndpi_process_packet(uint8_t * const args,
return;
} else if (flow_to_process->ndpi_flow->num_processed_pkts == 0xFE) {
/* last chance to guess something, better then nothing */
uint8_t protocol_was_guessed = 0;
flow_to_process->guessed_protocol =
ndpi_detection_giveup(workflow->ndpi_struct,
flow_to_process->ndpi_flow,
&protocol_was_guessed);
if (protocol_was_guessed != 0) {
flow_to_process->ndpi_flow);
if (flow_to_process->ndpi_flow->protocol_was_guessed != 0) {
printf("[%8llu, %d, %4d][GUESSED] protocol: %s | app protocol: %s | category: %s\n",
workflow->packets_captured,
reader_thread->array_index,
Expand Down
33 changes: 13 additions & 20 deletions example/reader_util.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1145,7 +1145,7 @@ static void dump_flow_fingerprint(struct ndpi_workflow * workflow,
ndpi_serialize_string_uint32(&serializer, "srv_port", ntohs(flow->dst_port));
ndpi_serialize_string_string(&serializer, "proto",
ndpi_protocol2name(workflow->ndpi_struct,
flow->detected_protocol,
flow->detected_protocol.proto,
buf, sizeof(buf)));

if(flow->server_hostname)
Expand Down Expand Up @@ -1192,7 +1192,7 @@ static void process_ndpi_monitoring_info(struct ndpi_flow_info *flow) {
return;

if(flow->monitoring_state == 0 &&
flow->ndpi_flow->monitoring) {
flow->ndpi_flow->state == NDPI_STATE_MONITORING) {
/* We just moved to monitoring state */
flow->monitoring_state = 1;
flow->num_packets_before_monitoring = flow->ndpi_flow->packet_direction_complete_counter[0] + flow->ndpi_flow->packet_direction_complete_counter[1];
Expand Down Expand Up @@ -1704,7 +1704,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
ndpi_serialize_string_uint32(&flow->ndpi_flow_serializer, "detection_completed", flow->detection_completed);
ndpi_serialize_string_uint32(&flow->ndpi_flow_serializer, "check_extra_packets", flow->check_extra_packets);

if(flow->ndpi_flow->monitoring) {
if(flow->ndpi_flow->state == NDPI_STATE_MONITORING) {
serialize_monitoring_metadata(flow);
}

Expand Down Expand Up @@ -2018,26 +2018,19 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
ipsize, time_ms, &input_info);
if(monitoring_enabled)
process_ndpi_monitoring_info(flow);
enough_packets |= ndpi_flow->fail_with_unknown;
if(enough_packets || (flow->detected_protocol.proto.app_protocol != NDPI_PROTOCOL_UNKNOWN)) {
if((!enough_packets)
&& ndpi_extra_dissection_possible(workflow->ndpi_struct, ndpi_flow))
; /* Wait for further metadata */
else {
/* New protocol detected or give up */
flow->detection_completed = 1;

if(flow->detected_protocol.proto.app_protocol == NDPI_PROTOCOL_UNKNOWN) {
u_int8_t proto_guessed;

flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct, flow->ndpi_flow,
&proto_guessed);
if(proto_guessed) workflow->stats.guessed_flow_protocols++;
}
if(flow->detected_protocol.state == NDPI_STATE_CLASSIFIED ||
enough_packets) {

flow->detection_completed = 1;

process_ndpi_collected_info(workflow, flow);
if(flow->detected_protocol.state != NDPI_STATE_CLASSIFIED) {
flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct, flow->ndpi_flow);
}

if(flow->ndpi_flow->protocol_was_guessed) workflow->stats.guessed_flow_protocols++;
process_ndpi_collected_info(workflow, flow);
}

/* Let's try to save client-server direction */
flow->current_pkt_from_client_to_server = input_info.in_pkt_dir;

Expand Down
8 changes: 4 additions & 4 deletions fuzz/fuzz_config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
FuzzedDataProvider fuzzed_data(data, size);
struct ndpi_detection_module_struct *ndpi_info_mod;
struct ndpi_flow_struct flow;
u_int8_t protocol_was_guessed, unused;
u_int8_t unused;
u_int32_t i, ret;
u_int16_t bool_value;
struct ndpi_lru_cache_stats lru_stats;
Expand Down Expand Up @@ -831,15 +831,15 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
ndpi_detection_get_l4(pkt.data(), pkt.size(), &l4_return, &l4_len_return, &l4_protocol_return, NDPI_DETECTION_ONLY_IPV4);

ndpi_detection_process_packet(ndpi_info_mod, &flow, pkt.data(), pkt.size(), 0, &input_info);
p = ndpi_detection_giveup(ndpi_info_mod, &flow, &protocol_was_guessed);
p = ndpi_detection_giveup(ndpi_info_mod, &flow);

assert(p.proto.master_protocol == ndpi_get_flow_masterprotocol(&flow));
assert(p.proto.app_protocol == ndpi_get_flow_appprotocol(&flow));
assert(p.category == ndpi_get_flow_category(&flow));
ndpi_is_master_only_protocol(ndpi_info_mod, p.proto.app_protocol);
ndpi_normalize_protocol(ndpi_info_mod, &p.proto);
assert(ndpi_stack_get_upper_proto(&p.protocol_stack) == ndpi_get_upper_proto(p));
assert(ndpi_stack_get_lower_proto(&p.protocol_stack) == ndpi_get_lower_proto(p));
assert(ndpi_stack_get_upper_proto(&p.protocol_stack) == ndpi_get_upper_proto(p.proto));
assert(ndpi_stack_get_lower_proto(&p.protocol_stack) == ndpi_get_lower_proto(p.proto));
ndpi_get_flow_error_code(&flow);
ndpi_get_flow_risk_info(&flow, out, sizeof(out), 1);
ndpi_get_flow_ndpi_proto(&flow, &p2);
Expand Down
4 changes: 1 addition & 3 deletions fuzz/fuzz_ndpi_reader.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,8 @@ static void node_cleanup_walker(const void *node, ndpi_VISIT which, int depth, v

if((which == ndpi_preorder) || (which == ndpi_leaf)) { /* Avoid walking the same node multiple times */
if((!flow->detection_completed) && flow->ndpi_flow) {
u_int8_t proto_guessed;

flow->detected_protocol = ndpi_detection_giveup(workflow->ndpi_struct,
flow->ndpi_flow, &proto_guessed);
flow->ndpi_flow);
}

process_ndpi_collected_info(workflow, flow);
Expand Down
15 changes: 3 additions & 12 deletions fuzz/fuzz_process_packet.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,6 @@ static ndpi_serializer json_serializer = {};
static ndpi_serializer csv_serializer = {};

int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
uint8_t protocol_was_guessed;

if (ndpi_info_mod == NULL) {
fuzz_init_detection_module(&ndpi_info_mod, NULL);

Expand All @@ -22,19 +20,12 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
memset(&ndpi_flow, 0, SIZEOF_FLOW_STRUCT);
ndpi_protocol detected_protocol =
ndpi_detection_process_packet(ndpi_info_mod, &ndpi_flow, Data, Size, 0, NULL);
ndpi_protocol guessed_protocol =
ndpi_detection_giveup(ndpi_info_mod, &ndpi_flow, &protocol_was_guessed);
detected_protocol = ndpi_detection_giveup(ndpi_info_mod, &ndpi_flow);

ndpi_reset_serializer(&json_serializer);
ndpi_reset_serializer(&csv_serializer);
if (protocol_was_guessed == 0)
{
ndpi_dpi2json(ndpi_info_mod, &ndpi_flow, detected_protocol, &json_serializer);
ndpi_dpi2json(ndpi_info_mod, &ndpi_flow, detected_protocol, &csv_serializer);
} else {
ndpi_dpi2json(ndpi_info_mod, &ndpi_flow, guessed_protocol, &json_serializer);
ndpi_dpi2json(ndpi_info_mod, &ndpi_flow, guessed_protocol, &csv_serializer);
}
ndpi_dpi2json(ndpi_info_mod, &ndpi_flow, detected_protocol, &json_serializer);
ndpi_dpi2json(ndpi_info_mod, &ndpi_flow, detected_protocol, &csv_serializer);
ndpi_free_flow_data(&ndpi_flow);

return 0;
Expand Down
3 changes: 1 addition & 2 deletions python/ndpi/ndpi.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,7 @@ def process_packet(self, flow, packet, packet_time_ms, input_info):

def giveup(self, flow):
p = lib.ndpi_detection_giveup(self._detection_module,
flow.C,
ffi.new("uint8_t*", 0))
flow.C)
return ndpi_protocol(C=p,
master_protocol=p.proto.master_protocol,
app_protocol=p.proto.app_protocol,
Expand Down
3 changes: 1 addition & 2 deletions python/ndpi/ndpi_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,7 @@
const u_int64_t packet_time_ms,
struct ndpi_flow_input_info *input_info);
ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int8_t *protocol_was_guessed);
struct ndpi_flow_struct *flow);
void ndpi_py_setup_detection_module(struct ndpi_detection_module_struct *mod);
struct ndpi_flow_struct * ndpi_py_initialize_flow(void);
char* ndpi_protocol2name(struct ndpi_detection_module_struct *ndpi_mod, ndpi_protocol proto, char *buf, u_int buf_len);
Expand Down
20 changes: 8 additions & 12 deletions src/include/ndpi_api.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,13 +223,11 @@ extern "C" {
*
* @par ndpi_struct = the detection module
* @par flow = the flow given for the detection module
* @par protocol_was_guessed = 1 if the protocol was guesses (requires enable_guess = 1), 0 otherwise
* @return the detected protocol even if the flow is not completed;
*
*/
ndpi_protocol ndpi_detection_giveup(struct ndpi_detection_module_struct *ndpi_struct,
struct ndpi_flow_struct *flow,
u_int8_t *protocol_was_guessed);
struct ndpi_flow_struct *flow);

/**
* Processes one packet and returns the ID of the detected protocol.
Expand Down Expand Up @@ -415,26 +413,26 @@ extern "C" {
* Write the protocol name in the buffer -buf- as master_protocol.protocol
*
* @par ndpi_mod = the detection module
* @par proto = the struct ndpi_protocol contain the protocols name
* @par proto = the struct ndpi_master_app_protocol contain the protocols name
* @par buf = the buffer to write the name of the protocols
* @par buf_len = the length of the buffer
* @return the buffer contains the master_protocol and protocol name
*
*/
char* ndpi_protocol2name(struct ndpi_detection_module_struct *ndpi_mod,
ndpi_protocol proto, char *buf, u_int buf_len);
ndpi_master_app_protocol proto, char *buf, u_int buf_len);

/**
* Same as ndpi_protocol2name() with the difference that the numeric protocol
* name is returned
*
* @par proto = the struct ndpi_protocol contain the protocols name
* @par proto = the struct ndpi_master_app_protocol contain the protocols name
* @par buf = the buffer to write the name of the protocols
* @par buf_len = the length of the buffer
* @return the buffer contains the master_protocol and protocol name
*
*/
char* ndpi_protocol2id(ndpi_protocol proto, char *buf, u_int buf_len);
char* ndpi_protocol2id(ndpi_master_app_protocol proto, char *buf, u_int buf_len);

/**
* Find out if a given category is custom/user-defined
Expand Down Expand Up @@ -975,8 +973,8 @@ extern "C" {
ndpi_l4_proto_info ndpi_get_l4_proto_info(struct ndpi_detection_module_struct *ndpi_struct, u_int16_t ndpi_proto_id);
const char* ndpi_get_l4_proto_name(ndpi_l4_proto_info proto);

u_int16_t ndpi_get_lower_proto(ndpi_protocol proto);
u_int16_t ndpi_get_upper_proto(ndpi_protocol proto);
u_int16_t ndpi_get_lower_proto(ndpi_master_app_protocol proto);
u_int16_t ndpi_get_upper_proto(ndpi_master_app_protocol proto);
bool ndpi_is_proto(ndpi_master_app_protocol proto, u_int16_t p);
bool ndpi_is_proto_unknown(ndpi_master_app_protocol proto);
bool ndpi_is_proto_equals(ndpi_master_app_protocol to_check, ndpi_master_app_protocol to_match, bool exact_match_only);
Expand Down Expand Up @@ -1042,8 +1040,6 @@ extern "C" {
int ndpi_flowv6_flow_hash(u_int8_t l4_proto, const struct ndpi_in6_addr *src_ip, const struct ndpi_in6_addr *dst_ip,
u_int16_t src_port, u_int16_t dst_port, u_int8_t icmp_type, u_int8_t icmp_code,
u_char *hash_buf, u_int8_t hash_buf_len);
u_int8_t ndpi_extra_dissection_possible(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow);
u_int8_t ndpi_is_safe_ssl_cipher(u_int32_t cipher);
const char* ndpi_cipher2str(u_int32_t cipher, char unknown_cipher[8]);
const char* ndpi_tunnel2str(ndpi_packet_tunnel tt);
Expand Down Expand Up @@ -1136,7 +1132,7 @@ extern "C" {
u_int8_t ndpi_is_public_ipv4(u_int32_t a /* host byte order */);
u_int64_t ndpi_htonll(u_int64_t v);
u_int64_t ndpi_ntohll(u_int64_t v);
u_int8_t ndpi_is_encrypted_proto(struct ndpi_detection_module_struct *ndpi_str, ndpi_protocol proto);
u_int8_t ndpi_is_encrypted_proto(struct ndpi_detection_module_struct *ndpi_str, ndpi_master_app_protocol proto);

/* DGA */
int ndpi_check_dga_name(struct ndpi_detection_module_struct *ndpi_str,
Expand Down
Loading
Loading