tokenize some regexp patterns

Author: atuchin-m

src/filters/network.rs

@@ -311,6 +311,7 @@ pub enum FilterPart {
     AnyOf(Vec<String>),
 }
 
+#[derive(Debug, PartialEq)]
 pub enum FilterTokens {
     Empty,
     OptDomains(Vec<Hash>),
@@ -929,6 +930,17 @@ impl NetworkFilter {
             if let Some(hostname) = self.hostname.as_ref() {
                 utils::tokenize_to(hostname, &mut tokens);
             }
+        } else if let Some(hostname) = self.hostname.as_ref() {
+            // For hostname regex patterns, try to extract literal tokens from the beginning
+            // Only tokenize if it looks like <literal>.<pattern> where we can extract the literal part
+            if let Some(dot_pos) = hostname.find('.') {
+                let prefix = &hostname[..dot_pos];
+                // Allow alphanumeric and hyphens (valid hostname chars)
+                // Minimum 2 chars to avoid overly generic tokens
+                if prefix.len() >= 2 && prefix.chars().all(|c| c.is_alphanumeric() || c == '-') {
+                    tokens.push(utils::fast_hash(prefix));
+                }
+            }
         }
 
         if tokens.is_empty() && self.mask.contains(NetworkFilterMask::IS_REMOVEPARAM) {

tests/unit/engine.rs

@@ -183,7 +183,7 @@ mod tests {
     fn deserialization_generate_simple() {
         let mut engine = Engine::from_rules(["ad-banner"], Default::default());
         let data = engine.serialize().to_vec();
-        const EXPECTED_HASH: u64 = 10945714988765761881;
+        const EXPECTED_HASH: u64 = 4966705697069852975;
         assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}");
         engine.deserialize(&data).unwrap();
     }
@@ -193,7 +193,7 @@ mod tests {
         let mut engine = Engine::from_rules(["ad-banner$tag=abc"], Default::default());
         engine.use_tags(&["abc"]);
         let data = engine.serialize().to_vec();
-        const EXPECTED_HASH: u64 = 4608037684406751718;
+        const EXPECTED_HASH: u64 = 13197424322352491802;
         assert_eq!(hash(&data), EXPECTED_HASH, "{HASH_MISMATCH_MSG}");
         engine.deserialize(&data).unwrap();
     }
@@ -237,9 +237,9 @@ mod tests {
             );
         }
         let expected_hash: u64 = if cfg!(feature = "css-validation") {
-            9439492009815519037
+            9241597848569736555
         } else {
-            14803842039735157685
+            15978625802684888070
         };
 
         assert_eq!(hash(&data), expected_hash, "{HASH_MISMATCH_MSG}");

tests/unit/filters/network.rs

@@ -1186,4 +1186,16 @@ mod parse_tests {
         defaults.opt_domains = Some(vec![utils::fast_hash("auth.wi-fi.ru")]);
         assert_eq!(defaults, NetworkFilterBreakdown::from(&filter));
     }
+
+    #[test]
+    fn test_simple_pattern_tokenization() {
+        let rule = "||primewire.*/sw$script,1p";
+        let filter =
+            NetworkFilter::parse(rule, true, crate::lists::ParseOptions::default()).unwrap();
+        let tokens = filter.get_tokens_optimized();
+        assert_eq!(
+            tokens,
+            crate::filters::network::FilterTokens::Other(vec![utils::fast_hash("primewire")])
+        );
+    }
 }