Merge branch 'neon-mmd:rolling' into hf-rolling

Cargo.lock

@@ -136,6 +136,7 @@ checksum = "28f32d40287d3f402ae0028a9d54bef51af15c8769492826a69d28f81893151d"
 dependencies = [
  "futures-core",
  "tokio 1.36.0",
+ "tokio-uring",
 ]
 
 [[package]]
@@ -150,8 +151,9 @@ dependencies = [
  "futures-core",
  "futures-util",
  "mio 0.8.11",
- "socket2",
+ "socket2 0.5.6",
  "tokio 1.36.0",
+ "tokio-uring",
  "tracing",
 ]
 
@@ -211,7 +213,7 @@ dependencies = [
  "serde_json",
  "serde_urlencoded 0.7.1",
  "smallvec 1.13.1",
- "socket2",
+ "socket2 0.5.6",
  "time 0.3.34",
  "url 2.5.0",
 ]
@@ -243,6 +245,12 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
 
+[[package]]
+name = "adler32"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
+
 [[package]]
 name = "aead"
 version = "0.5.2"
@@ -1667,7 +1675,7 @@ dependencies = [
  "httpdate",
  "itoa 1.0.10",
  "pin-project-lite",
- "socket2",
+ "socket2 0.5.6",
  "tokio 1.36.0",
  "tower-service",
  "tracing",
@@ -1762,6 +1770,16 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "io-uring"
+version = "0.5.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd1e1a01cfb924fd8c5c43b6827965db394f5a3a16c599ce03452266e1cf984c"
+dependencies = [
+ "bitflags 1.3.2",
+ "libc",
+]
+
 [[package]]
 name = "iovec"
 version = "0.1.4"
@@ -1828,6 +1846,16 @@ dependencies = [
  "winapi-build",
 ]
 
+[[package]]
+name = "keyword_extraction"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c25710ba2c50e4762b267b7387a989d8d1a8235f5cf26cd84e34aac30b263140"
+dependencies = [
+ "regex",
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "language-tags"
 version = "0.3.2"
@@ -1846,6 +1874,26 @@ version = "0.2.153"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
 
+[[package]]
+name = "libflate"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ff4ae71b685bbad2f2f391fe74f6b7659a34871c08b210fdc039e43bee07d18"
+dependencies = [
+ "adler32",
+ "crc32fast",
+ "libflate_lz77",
+]
+
+[[package]]
+name = "libflate_lz77"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a52d3a8bfc85f250440e4424db7d857e241a3aebbbe301f3eb606ab15c39acbf"
+dependencies = [
+ "rle-decode-fast",
+]
+
 [[package]]
 name = "libmimalloc-sys"
 version = "0.1.35"
@@ -3017,6 +3065,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rle-decode-fast"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
+
 [[package]]
 name = "rustc-demangle"
 version = "0.1.23"
@@ -3127,6 +3181,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "scoped-tls"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294"
+
 [[package]]
 name = "scopeguard"
 version = "1.2.0"
@@ -3372,6 +3432,16 @@ dependencies = [
  "serde",
 ]
 
+[[package]]
+name = "socket2"
+version = "0.4.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f7916fc008ca5542385b89a3d3ce689953c143e9304a9bf8beec1de48994c0d"
+dependencies = [
+ "libc",
+ "winapi 0.3.9",
+]
+
 [[package]]
 name = "socket2"
 version = "0.5.6"
@@ -3403,6 +3473,15 @@ version = "1.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
 
+[[package]]
+name = "stop-words"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8500024d809de02ecbf998472b7bed3c4fca380df2be68917f6a473bdb28ddcc"
+dependencies = [
+ "serde_json",
+]
+
 [[package]]
 name = "string"
 version = "0.2.1"
@@ -3579,6 +3658,25 @@ dependencies = [
  "utf-8",
 ]
 
+[[package]]
+name = "thesaurus"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e33ea271e53da683cd3439c04ff3b734f3d6052ea33a65ec9e0fa89a4f96369"
+dependencies = [
+ "lazy_static",
+ "thesaurus-moby",
+]
+
+[[package]]
+name = "thesaurus-moby"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28f7806d5dbe7d9b627e332f88269a014a6a1d40ec411d4ea66cb702aabce4cf"
+dependencies = [
+ "libflate",
+]
+
 [[package]]
 name = "thousands"
 version = "0.2.0"
@@ -3685,7 +3783,7 @@ dependencies = [
  "parking_lot 0.12.1",
  "pin-project-lite",
  "signal-hook-registry",
- "socket2",
+ "socket2 0.5.6",
  "tokio-macros",
  "windows-sys 0.48.0",
 ]
@@ -3836,6 +3934,20 @@ dependencies = [
  "tokio-executor",
 ]
 
+[[package]]
+name = "tokio-uring"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d5e02bb137e030b3a547c65a3bd2f1836d66a97369fdcc69034002b10e155ef"
+dependencies = [
+ "io-uring",
+ "libc",
+ "scoped-tls",
+ "slab",
+ "socket2 0.4.10",
+ "tokio 1.36.0",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.7.10"
@@ -3942,6 +4054,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
+
 [[package]]
 name = "unicode-width"
 version = "0.1.11"
@@ -4157,7 +4275,7 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1"
 
 [[package]]
 name = "websurfx"
-version = "1.10.9"
+version = "1.12.1"
 dependencies = [
  "actix-cors",
  "actix-files",
@@ -4177,6 +4295,7 @@ dependencies = [
  "error-stack",
  "fake-useragent",
  "futures 0.3.30",
+ "keyword_extraction",
  "lightningcss",
  "log",
  "maud",
@@ -4192,7 +4311,9 @@ dependencies = [
  "serde",
  "serde_json",
  "smallvec 1.13.1",
+ "stop-words",
  "tempfile",
+ "thesaurus",
  "tokio 1.36.0",
 ]
 

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "websurfx"
-version = "1.10.9"
+version = "1.12.1"
 edition = "2021"
 description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
 repository = "https://github.com/neon-mmd/websurfx"
@@ -13,45 +13,93 @@ bench = false
 path = "src/bin/websurfx.rs"
 
 [dependencies]
-reqwest = {version="0.11.24", default-features=false, features=["rustls-tls","brotli", "gzip"]}
-tokio = {version="1.32.0",features=["rt-multi-thread","macros", "fs", "io-util"], default-features = false}
-serde = {version="1.0.196", default-features=false, features=["derive"]}
-serde_json = {version="1.0.109", default-features=false}
-maud = {version="0.25.0", default-features=false, features=["actix-web"]}
-scraper = {version="0.18.1", default-features = false}
-actix-web = {version="4.4.0", features = ["cookies", "macros", "compress-brotli"], default-features=false}
-actix-files = {version="0.6.5", default-features=false}
-actix-cors = {version="0.7.0", default-features=false}
-fake-useragent = {version="0.1.3", default-features=false}
-env_logger = {version="0.11.1", default-features=false}
-log = {version="0.4.21", default-features=false}
-mlua = {version="0.9.1", features=["luajit", "vendored"], default-features=false}
-redis = {version="0.24.0", features=["tokio-comp","connection-manager"], default-features = false, optional = true}
-blake3 = {version="1.5.0", default-features=false}
-error-stack = {version="0.4.0", default-features=false, features=["std"]}
-async-trait = {version="0.1.76", default-features=false}
-regex = {version="1.9.4", features=["perf"], default-features = false}
-smallvec = {version="1.13.1", features=["union", "serde"], default-features=false}
-futures = {version="0.3.30", default-features=false, features=["alloc"]}
-dhat = {version="0.3.2", optional = true, default-features=false}
+reqwest = { version = "0.11.24", default-features = false, features = [
+    "rustls-tls",
+    "brotli",
+    "gzip",
+] }
+tokio = { version = "1.32.0", features = [
+    "rt-multi-thread",
+    "macros",
+    "fs",
+    "io-util",
+], default-features = false }
+serde = { version = "1.0.196", default-features = false, features = ["derive"] }
+serde_json = { version = "1.0.109", default-features = false }
+maud = { version = "0.25.0", default-features = false, features = [
+    "actix-web",
+] }
+scraper = { version = "0.18.1", default-features = false }
+actix-web = { version = "4.4.0", features = [
+    "cookies",
+    "macros",
+    "compress-brotli",
+], default-features = false }
+actix-files = { version = "0.6.5", default-features = false }
+actix-cors = { version = "0.7.0", default-features = false }
+fake-useragent = { version = "0.1.3", default-features = false }
+env_logger = { version = "0.11.1", default-features = false }
+log = { version = "0.4.21", default-features = false }
+mlua = { version = "0.9.1", features = [
+    "luajit",
+    "vendored",
+], default-features = false }
+redis = { version = "0.24.0", features = [
+    "tokio-comp",
+    "connection-manager",
+], default-features = false, optional = true }
+blake3 = { version = "1.5.0", default-features = false }
+error-stack = { version = "0.4.0", default-features = false, features = [
+    "std",
+] }
+async-trait = { version = "0.1.76", default-features = false }
+regex = { version = "1.9.4", features = ["perf"], default-features = false }
+smallvec = { version = "1.13.1", features = [
+    "union",
+    "serde",
+], default-features = false }
+futures = { version = "0.3.30", default-features = false, features = ["alloc"] }
+dhat = { version = "0.3.2", optional = true, default-features = false }
 mimalloc = { version = "0.1.38", default-features = false }
-async-once-cell = {version="0.5.3", default-features=false}
-actix-governor = {version="0.5.0", default-features=false}
-mini-moka = { version="0.10", optional = true, default-features=false, features=["sync"]}
-async-compression = { version = "0.4.6", default-features = false, features=["brotli","tokio"], optional=true}
-chacha20poly1305={version="0.10.1", default-features=false, features=["alloc","getrandom"], optional=true}
-chacha20 = {version="0.9.1", default-features=false, optional=true}
-base64 = {version="0.21.5", default-features=false, features=["std"], optional=true}
-cfg-if = {version="1.0.0", default-features=false,optional=true}
+async-once-cell = { version = "0.5.3", default-features = false }
+actix-governor = { version = "0.5.0", default-features = false }
+mini-moka = { version = "0.10", optional = true, default-features = false, features = [
+    "sync",
+] }
+async-compression = { version = "0.4.6", default-features = false, features = [
+    "brotli",
+    "tokio",
+], optional = true }
+chacha20poly1305 = { version = "0.10.1", default-features = false, features = [
+    "alloc",
+    "getrandom",
+], optional = true }
+chacha20 = { version = "0.9.1", default-features = false, optional = true }
+base64 = { version = "0.21.5", default-features = false, features = [
+    "std",
+], optional = true }
+cfg-if = { version = "1.0.0", default-features = false, optional = true }
+keyword_extraction = { version = "1.3.0", default-features = false, features = [
+    "tf_idf",
+
+
+] }
+
+stop-words = { version = "0.8.0", default-features = false, features = ["iso"] }
+thesaurus = { version = "0.5.2", default-features = false, optional = true, features = [
+    "moby",
+] }
 
 [dev-dependencies]
-rusty-hook = {version="^0.11.2", default-features=false}
-criterion = {version="0.5.1", default-features=false}
-tempfile = {version="3.10.1", default-features=false}
+rusty-hook = { version = "^0.11.2", default-features = false }
+criterion = { version = "0.5.1", default-features = false }
+tempfile = { version = "3.10.1", default-features = false }
 
 [build-dependencies]
-lightningcss = {version="1.0.0-alpha.52", default-features=false, features=["grid"]}
-minify-js = {version="0.6.0", default-features=false}
+lightningcss = { version = "1.0.0-alpha.52", default-features = false, features = [
+    "grid",
+] }
+minify-js = { version = "0.6.0", default-features = false }
 
 [profile.dev]
 opt-level = 0
@@ -72,19 +120,21 @@ debug = false # This should only be commented when testing with dhat profiler
 split-debuginfo = '...'
 debug-assertions = false
 overflow-checks = false
-lto = true
+lto = 'thin'
 panic = 'abort'
 incremental = false
 codegen-units = 1
 rpath = false
-strip = "debuginfo"
+strip = "symbols"
 
 [features]
+use-synonyms-search = ["thesaurus/static"]
 default = ["memory-cache"]
-dhat-heap = ["dep:dhat"] 
+dhat-heap = ["dep:dhat"]
 memory-cache = ["dep:mini-moka"]
-redis-cache = ["dep:redis","dep:base64"]
-compress-cache-results = ["dep:async-compression","dep:cfg-if"]
-encrypt-cache-results = ["dep:chacha20poly1305","dep:chacha20"]
-cec-cache-results = ["compress-cache-results","encrypt-cache-results"]
-
+redis-cache = ["dep:redis", "dep:base64"]
+compress-cache-results = ["dep:async-compression", "dep:cfg-if"]
+encrypt-cache-results = ["dep:chacha20poly1305", "dep:chacha20"]
+cec-cache-results = ["compress-cache-results", "encrypt-cache-results"]
+experimental-io-uring = ["actix-web/experimental-io-uring"]
+use-non-static-synonyms-search = ["thesaurus"]

public/static/colorschemes/rose-pine-dawn.css

@@ -0,0 +1,12 @@
+:root {
+  --background-color: #faf4ed;
+  --foreground-color: #575279;
+  --logo-color: #d7827e;
+  --color-one: #f2e9e1;
+  --color-two: #907aa9;
+  --color-three: #56949f;
+  --color-four: #ea9d34;
+  --color-five: #d7827e;
+  --color-six: #9893a5;
+  --color-seven: #575279;
+}

public/static/colorschemes/rose-pine-moon.css

@@ -0,0 +1,12 @@
+:root {
+  --background-color: #232136;
+  --foreground-color: #e0def4;
+  --logo-color: #ea9a97;
+  --color-one: #393552;
+  --color-two: #c4a7e7;
+  --color-three: #9ccfd8;
+  --color-four: #f6c177;
+  --color-five: #ea9a97;
+  --color-six: #6e6a86;
+  --color-seven: #e0def4;
+}

public/static/colorschemes/rose-pine.css

@@ -0,0 +1,12 @@
+:root {
+    --background-color: #191724;
+    --foreground-color: #e0def4;
+    --logo-color: #ebbcba;
+    --color-one: #26233a;
+    --color-two: #c4a7e7;
+    --color-three: #9ccfd8;
+    --color-four: #f6c177;
+    --color-five: #eb6f92;
+    --color-six: #6e6a86;
+    --color-seven: #e0def4;
+}

public/static/themes/simple.css

@@ -482,6 +482,7 @@ footer div {
 
 .about-container article .logo-container svg {
   width: clamp(200px, 530px, 815px);
+  color: var(--logo-color);
 }
 
 .about-container article .text-block {
@@ -858,4 +859,4 @@ input:checked + .slider::before {
   .feature-card {
     border-radius: 0;
   }
-}
+}

src/config/parser.rs

@@ -42,6 +42,10 @@ pub struct Config {
     /// It stores the level of safe search to be used for restricting content in the
     /// search results.
     pub safe_search: u8,
+    /// It stores the TCP connection keepalive duration in seconds.
+    pub tcp_connection_keepalive: u8,
+    /// It stores the pool idle connection timeout in seconds.
+    pub pool_idle_connection_timeout: u8,
 }
 
 impl Config {
@@ -131,6 +135,8 @@ impl Config {
             upstream_search_engines: globals
                 .get::<_, HashMap<String, bool>>("upstream_search_engines")?,
             request_timeout: globals.get::<_, u8>("request_timeout")?,
+            tcp_connection_keepalive: globals.get::<_, u8>("tcp_connection_keepalive")?,
+            pool_idle_connection_timeout: globals.get::<_, u8>("pool_idle_connection_timeout")?,
             threads,
             rate_limiter: RateLimiter {
                 number_of_requests: rate_limiter["number_of_requests"],

src/models/aggregation_models.rs

@@ -4,7 +4,11 @@
 use super::engine_models::EngineError;
 use serde::{Deserialize, Serialize};
 use smallvec::SmallVec;
-
+#[cfg(any(
+    feature = "use-synonyms-search",
+    feature = "use-non-static-synonyms-search"
+))]
+use thesaurus::synonyms;
 /// A named struct to store the raw scraped search results scraped search results from the
 /// upstream search engines before aggregating it.It derives the Clone trait which is needed
 /// to write idiomatic rust using `Iterators`.
@@ -20,6 +24,8 @@ pub struct SearchResult {
     pub description: String,
     /// The names of the upstream engines from which this results were provided.
     pub engine: SmallVec<[String; 0]>,
+    /// The td-tdf score of the result in regards to the title, url and description and the user's query
+    pub relevance_score: f32,
 }
 
 impl SearchResult {
@@ -37,9 +43,49 @@ impl SearchResult {
             title: title.to_owned(),
             url: url.to_owned(),
             description: description.to_owned(),
+            relevance_score: 0.0,
             engine: engine.iter().map(|name| name.to_string()).collect(),
         }
     }
+    /// calculates and update the relevance score of the current search.
+
+    /// # Arguments
+    ///
+    /// * query -  the query string  used to obtain the results
+    ///
+    ///
+
+    pub fn calculate_relevance(&mut self, query: &str) {
+        use stop_words::{get, LANGUAGE};
+        // when language settings can change to any of the ones supported on this crate: https://docs.rs/crate/stop-words/0.8.0
+        let documents = [
+            self.title.clone(),
+            self.url.clone(),
+            self.description.clone(),
+        ];
+
+        let stop_words = get(LANGUAGE::English);
+        let punctuation = [
+            ".".to_owned(),
+            ",".to_owned(),
+            ":".to_owned(),
+            ";".to_owned(),
+            "!".to_owned(),
+            "?".to_owned(),
+            "(".to_owned(),
+            ")".to_owned(),
+            "[".to_owned(),
+            "]".to_owned(),
+            "{".to_owned(),
+            "}".to_owned(),
+            "\"".to_owned(),
+            "'".to_owned(),
+            "<".to_owned(),
+            ">".to_owned(),
+        ];
+
+        self.relevance_score = calculate_tf_idf(query, &documents, &stop_words, &punctuation);
+    }
 
     /// A function which adds the engine name provided as a string into a vector of strings.
     ///
@@ -182,3 +228,53 @@ impl SearchResults {
         self.no_engines_selected = true;
     }
 }
+
+/// Helper function to calculate the tf-idf for the search query.
+/// <br> The approach is  as [`as`](https://en.wikipedia.org/wiki/Tf%E2%80%93idf).
+///  <br> Find a sample article about TF-IDF [`here`](https://medium.com/analytics-vidhya/tf-idf-term-frequency-technique-easiest-explanation-for-text-classification-in-nlp-with-code-8ca3912e58c3)
+/// ### Arguments
+/// * `query` -  a user's search query
+/// * `documents` -  a list of text used for comparision (url, title, description)
+/// * `stop_words` - A list of language specific stop words.
+/// * `punctuation` - list of punctuation symbols.
+/// ### Returns
+/// * `score` - The average tf-idf score of the word tokens (and synonyms) in the query
+fn calculate_tf_idf(
+    query: &str,
+    documents: &[String],
+    stop_words: &[String],
+    punctuation: &[String],
+) -> f32 {
+    use keyword_extraction::{
+        tf_idf::{TfIdf, TfIdfParams},
+        tokenizer::Tokenizer,
+    };
+
+    let params = TfIdfParams::UnprocessedDocuments(documents, stop_words, Some(punctuation));
+    let tf_idf = TfIdf::new(params);
+    let tokener = Tokenizer::new(query, stop_words, Some(punctuation));
+    let query_tokens = tokener.split_into_words();
+    let mut search_tokens = vec![];
+
+    for token in query_tokens {
+        #[cfg(any(
+            feature = "use-synonyms-search",
+            feature = "use-non-static-synonyms-search"
+        ))]
+        {
+            // find some synonyms and add them to the search  (from wordnet or moby if feature is enabled)
+            let synonyms = synonyms(&token);
+            search_tokens.extend(synonyms)
+        }
+        search_tokens.push(token);
+    }
+
+    let mut total_score = 0.0f32;
+    for token in search_tokens.iter() {
+        total_score += tf_idf.get_score(token);
+    }
+
+    let result = total_score / (search_tokens.len() as f32);
+
+    f32::from(!result.is_nan()) * result
+}

src/results/aggregator.rs

@@ -8,6 +8,7 @@ use crate::models::{
     aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
     engine_models::{EngineError, EngineHandler},
 };
+
 use error_stack::Report;
 use futures::stream::FuturesUnordered;
 use regex::Regex;
@@ -77,6 +78,11 @@ pub async fn aggregate(
     let client = CLIENT.get_or_init(|| {
         ClientBuilder::new()
             .timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
+            .pool_idle_timeout(Duration::from_secs(
+                config.pool_idle_connection_timeout as u64,
+            ))
+            .tcp_keepalive(Duration::from_secs(config.tcp_connection_keepalive as u64))
+            .connect_timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
             .https_only(true)
             .gzip(true)
             .brotli(true)
@@ -182,7 +188,17 @@ pub async fn aggregate(
         drop(blacklist_map);
     }
 
-    let results: Vec<SearchResult> = result_map.iter().map(|(_, value)| value.clone()).collect();
+    let mut results: Vec<SearchResult> = result_map
+        .iter()
+        .map(|(_, value)| {
+            let mut copy = value.clone();
+            if !copy.url.contains("temu.com") {
+                copy.calculate_relevance(query.as_str())
+            }
+            copy
+        })
+        .collect();
+    sort_search_results(&mut results);
 
     Ok(SearchResults::new(results, &engine_errors_info))
 }
@@ -231,7 +247,21 @@ pub async fn filter_with_lists(
 
     Ok(())
 }
-
+/// Sorts  SearchResults by relevance score.
+/// <br> sort_unstable is used as its faster,stability is not an issue on our side.
+/// For reasons why, check out [`this`](https://rust-lang.github.io/rfcs/1884-unstable-sort.html)
+///  # Arguments
+///  * `results` - A mutable slice or Vec of SearchResults
+///  
+fn sort_search_results(results: &mut [SearchResult]) {
+    results.sort_unstable_by(|a, b| {
+        use std::cmp::Ordering;
+
+        b.relevance_score
+            .partial_cmp(&a.relevance_score)
+            .unwrap_or(Ordering::Less)
+    })
+}
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -250,6 +280,7 @@ mod tests {
                 url: "https://www.example.com".to_owned(),
                 description: "This domain is for use in illustrative examples in documents."
                     .to_owned(),
+                relevance_score: 0.0,
                 engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
             },
         ));
@@ -260,6 +291,7 @@ mod tests {
                 url: "https://www.rust-lang.org/".to_owned(),
                 description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
                 engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
+                relevance_score:0.0
             },)
         );
 
@@ -300,6 +332,7 @@ mod tests {
                 description: "This domain is for use in illustrative examples in documents."
                     .to_owned(),
                 engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
+                relevance_score: 0.0,
             },
         ));
         map_to_be_filtered.push((
@@ -309,6 +342,7 @@ mod tests {
                 url: "https://www.rust-lang.org/".to_owned(),
                 description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
                 engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
+                relevance_score:0.0
             },
         ));
 
@@ -365,6 +399,7 @@ mod tests {
                 description: "This domain is for use in illustrative examples in documents."
                     .to_owned(),
                 engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
+                relevance_score: 0.0,
             },
         ));
 

websurfx/config.lua

@@ -10,6 +10,8 @@ production_use = false -- whether to use production mode or not (in other words
 -- if production_use is set to true
 -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
 request_timeout = 30 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds).
+tcp_connection_keepalive = 30 -- the amount of time the tcp connection should remain alive (or connected to the server). (value in seconds).
+pool_idle_connection_timeout = 30 -- timeout for the idle connections in the reqwest HTTP connection pool (value in seconds).
 rate_limiter = {
 	number_of_requests = 50, -- The number of request that are allowed within a provided time limit.
 	time_limit = 3, -- The time limit in which the quantity of requests that should be accepted.