Merge branch 'rolling' into feat-error-box-for-engine-errors

.cspell.json

@@ -11,6 +11,10 @@
   ],
   "language": "en",
   "noConfigSearch": true,
-  "words": ["megalinter", "oxsecurity"],
+  "words": [
+    "megalinter",
+    "oxsecurity",
+    "websurfx"
+  ],
   "version": "0.2"
 }

Cargo.lock

@@ -14,7 +14,7 @@ dependencies = [
  "futures-sink",
  "memchr",
  "pin-project-lite",
- "tokio 1.29.1",
+ "tokio 1.32.0",
  "tokio-util",
  "tracing",
 ]
@@ -90,7 +90,7 @@ dependencies = [
  "rand 0.8.5",
  "sha1",
  "smallvec 1.11.0",
- "tokio 1.29.1",
+ "tokio 1.32.0",
  "tokio-util",
  "tracing",
  "zstd",
@@ -102,8 +102,8 @@ version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
 dependencies = [
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -126,7 +126,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "15265b6b8e2347670eb363c47fc8c75208b4a4994b27192f345fcbe707804f3e"
 dependencies = [
  "futures-core",
- "tokio 1.29.1",
+ "tokio 1.32.0",
 ]
 
 [[package]]
@@ -142,8 +142,8 @@ dependencies = [
  "futures-util",
  "mio 0.8.8",
  "num_cpus",
- "socket2",
- "tokio 1.29.1",
+ "socket2 0.4.9",
+ "tokio 1.32.0",
  "tracing",
 ]
 
@@ -204,7 +204,7 @@ dependencies = [
  "serde_json",
  "serde_urlencoded 0.7.1",
  "smallvec 1.11.0",
- "socket2",
+ "socket2 0.4.9",
  "time 0.3.25",
  "url 2.4.0",
 ]
@@ -217,7 +217,7 @@ checksum = "2262160a7ae29e3415554a3f1fc04c764b1540c116aa524683208078b7a75bc9"
 dependencies = [
  "actix-router",
  "proc-macro2 1.0.66",
- "quote 1.0.32",
+ "quote 1.0.33",
  "syn 1.0.109",
 ]
 
@@ -261,9 +261,9 @@ dependencies = [
 
 [[package]]
 name = "aho-corasick"
-version = "1.0.2"
+version = "1.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41"
+checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a"
 dependencies = [
  "memchr",
 ]
@@ -297,9 +297,9 @@ checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
 
 [[package]]
 name = "anyhow"
-version = "1.0.72"
+version = "1.0.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
+checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
 
 [[package]]
 name = "askama_escape"
@@ -309,13 +309,13 @@ checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
 
 [[package]]
 name = "async-trait"
-version = "0.1.72"
+version = "0.1.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc6dde6e4ed435a4c1ee4e73592f5ba9da2151af10076cc04858746af9352d09"
+checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -386,9 +386,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
 [[package]]
 name = "bitflags"
-version = "2.3.3"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
+checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
 
 [[package]]
 name = "block-buffer"
@@ -475,9 +475,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
 [[package]]
 name = "cc"
-version = "1.0.81"
+version = "1.0.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6c6b2562119bf28c3439f7f02db99faf0aa1a8cdfe5772a2ee155d32227239f0"
+checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01"
 dependencies = [
  "jobserver",
  "libc",
@@ -533,18 +533,18 @@ dependencies = [
 
 [[package]]
 name = "clap"
-version = "4.3.19"
+version = "4.3.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fd304a20bff958a57f04c4e96a2e7594cc4490a0e809cbd48bb6437edaa452d"
+checksum = "03aef18ddf7d879c15ce20f04826ef8418101c7e528014c3eeea13321047dca3"
 dependencies = [
  "clap_builder",
 ]
 
 [[package]]
 name = "clap_builder"
-version = "4.3.19"
+version = "4.3.23"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01c6a3f08f1fe5662a35cfe393aec09c4df95f60ee93b7556505260f75eee9e1"
+checksum = "f8ce6fffb678c9b80a70b6b6de0aad31df727623a70fd9a842c30cd573e2fa98"
 dependencies = [
  "anstyle",
  "clap_lex",
@@ -810,8 +810,8 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
 dependencies = [
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -828,7 +828,7 @@ checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
 dependencies = [
  "convert_case",
  "proc-macro2 1.0.66",
- "quote 1.0.32",
+ "quote 1.0.33",
  "rustc_version 0.4.0",
  "syn 1.0.109",
 ]
@@ -956,7 +956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
+ "quote 1.0.33",
  "syn 1.0.109",
  "synstructure",
 ]
@@ -980,9 +980,9 @@ checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764"
 
 [[package]]
 name = "flate2"
-version = "1.0.26"
+version = "1.0.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b9429470923de8e8cbd4d2dc513535400b4b3fef0319fb5c4e1f520a7bef743"
+checksum = "c6c98ee8095e9d1dcbf2fcc6d95acccb90d1c81db1e44725c6a984b1dbdfb010"
 dependencies = [
  "crc32fast",
  "miniz_oxide",
@@ -1188,7 +1188,7 @@ dependencies = [
  "http 0.2.9",
  "indexmap",
  "slab",
- "tokio 1.29.1",
+ "tokio 1.32.0",
  "tokio-util",
  "tracing",
 ]
@@ -1250,7 +1250,7 @@ dependencies = [
  "mac",
  "markup5ever 0.11.0",
  "proc-macro2 1.0.66",
- "quote 1.0.32",
+ "quote 1.0.33",
  "syn 1.0.109",
 ]
 
@@ -1313,9 +1313,9 @@ checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
 
 [[package]]
 name = "httpdate"
-version = "1.0.2"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
 [[package]]
 name = "humantime"
@@ -1370,8 +1370,8 @@ dependencies = [
  "httpdate",
  "itoa 1.0.9",
  "pin-project-lite",
- "socket2",
- "tokio 1.29.1",
+ "socket2 0.4.9",
+ "tokio 1.32.0",
  "tower-service",
  "tracing",
  "want 0.3.1",
@@ -1399,7 +1399,7 @@ dependencies = [
  "bytes 1.4.0",
  "hyper 0.14.27",
  "native-tls",
- "tokio 1.29.1",
+ "tokio 1.32.0",
  "tokio-native-tls",
 ]
 
@@ -1583,9 +1583,9 @@ dependencies = [
 
 [[package]]
 name = "log"
-version = "0.4.19"
+version = "0.4.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4"
+checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
 
 [[package]]
 name = "mac"
@@ -1817,9 +1817,9 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
 
 [[package]]
 name = "openssl"
-version = "0.10.55"
+version = "0.10.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d"
+checksum = "729b745ad4a5575dd06a3e1af1414bd330ee561c01b3899eb584baeaa8def17e"
 dependencies = [
  "bitflags 1.3.2",
  "cfg-if 1.0.0",
@@ -1837,8 +1837,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -1849,9 +1849,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.90"
+version = "0.9.91"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6"
+checksum = "866b5f16f90776b9bb8dc1e1802ac6f0513de3a7a7465867bfbc563dc737faac"
 dependencies = [
  "cc",
  "libc",
@@ -1955,8 +1955,8 @@ dependencies = [
  "pest",
  "pest_meta",
  "proc-macro2 1.0.66",
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -2057,8 +2057,8 @@ dependencies = [
  "phf_generator 0.11.2",
  "phf_shared 0.11.2",
  "proc-macro2 1.0.66",
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -2090,9 +2090,9 @@ dependencies = [
 
 [[package]]
 name = "pin-project-lite"
-version = "0.2.11"
+version = "0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c516611246607d0c04186886dbb3a754368ef82c79e9827a802c6d836dd111c"
+checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05"
 
 [[package]]
 name = "pin-utils"
@@ -2185,9 +2185,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.32"
+version = "1.0.33"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
 dependencies = [
  "proc-macro2 1.0.66",
 ]
@@ -2361,16 +2361,16 @@ dependencies = [
 
 [[package]]
 name = "redis"
-version = "0.23.1"
+version = "0.23.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff5d95dd18a4d76650f0c2607ed8ebdbf63baf9cb934e1c233cd220c694db1d7"
+checksum = "ffd6543a7bc6428396845f6854ccf3d1ae8823816592e2cbe74f20f50f209d02"
 dependencies = [
  "combine",
  "itoa 1.0.9",
  "percent-encoding 2.3.0",
  "ryu",
  "sha1_smol",
- "socket2",
+ "socket2 0.4.9",
  "url 2.4.0",
 ]
 
@@ -2479,7 +2479,7 @@ dependencies = [
  "serde",
  "serde_json",
  "serde_urlencoded 0.7.1",
- "tokio 1.29.1",
+ "tokio 1.32.0",
  "tokio-native-tls",
  "tower-service",
  "url 2.4.0",
@@ -2539,11 +2539,11 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.38.7"
+version = "0.38.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "172891ebdceb05aa0005f533a6cbfca599ddd7d966f6f5d4d9b2e70478e70399"
+checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f"
 dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.0",
  "errno",
  "libc",
  "linux-raw-sys",
@@ -2648,7 +2648,7 @@ version = "0.25.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06"
 dependencies = [
- "bitflags 2.3.3",
+ "bitflags 2.4.0",
  "cssparser",
  "derive_more",
  "fxhash",
@@ -2684,29 +2684,29 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 
 [[package]]
 name = "serde"
-version = "1.0.182"
+version = "1.0.183"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bdb30a74471f5b7a1fa299f40b4bf1be93af61116df95465b2b5fc419331e430"
+checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.182"
+version = "1.0.183"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f4c2c6ea4bc09b5c419012eafcdb0fcef1d9119d626c8f3a0708a5b92d38a70"
+checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.104"
+version = "1.0.105"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "076066c5f1078eac5b722a31827a8832fe108bed65dfa75e233c89f8206e976c"
+checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"
 dependencies = [
  "itoa 1.0.9",
  "ryu",
@@ -2829,6 +2829,16 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
+[[package]]
+name = "socket2"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"
+dependencies = [
+ "libc",
+ "windows-sys",
+]
+
 [[package]]
 name = "stable_deref_trait"
 version = "1.2.0"
@@ -2882,7 +2892,7 @@ dependencies = [
  "phf_generator 0.7.24",
  "phf_shared 0.7.24",
  "proc-macro2 1.0.66",
- "quote 1.0.32",
+ "quote 1.0.33",
  "string_cache_shared",
 ]
 
@@ -2895,7 +2905,7 @@ dependencies = [
  "phf_generator 0.10.0",
  "phf_shared 0.10.0",
  "proc-macro2 1.0.66",
- "quote 1.0.32",
+ "quote 1.0.33",
 ]
 
 [[package]]
@@ -2922,18 +2932,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
+ "quote 1.0.33",
  "unicode-ident",
 ]
 
 [[package]]
 name = "syn"
-version = "2.0.28"
+version = "2.0.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567"
+checksum = "c324c494eba9d92503e6f1ef2e6df781e78f6a7705a0202d9801b198807d518a"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
+ "quote 1.0.33",
  "unicode-ident",
 ]
 
@@ -2944,16 +2954,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
+ "quote 1.0.33",
  "syn 1.0.109",
  "unicode-xid 0.2.4",
 ]
 
 [[package]]
 name = "tempfile"
-version = "3.7.0"
+version = "3.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998"
+checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef"
 dependencies = [
  "cfg-if 1.0.0",
  "fastrand",
@@ -2984,22 +2994,22 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.44"
+version = "1.0.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90"
+checksum = "97a802ec30afc17eee47b2855fc72e0c4cd62be9b4efe6591edde0ec5bd68d8f"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.44"
+version = "1.0.47"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96"
+checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -3087,11 +3097,10 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.29.1"
+version = "1.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da"
+checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
 dependencies = [
- "autocfg 1.1.0",
  "backtrace",
  "bytes 1.4.0",
  "libc",
@@ -3100,7 +3109,7 @@ dependencies = [
  "parking_lot 0.12.1",
  "pin-project-lite",
  "signal-hook-registry",
- "socket2",
+ "socket2 0.5.3",
  "tokio-macros",
  "windows-sys",
 ]
@@ -3154,8 +3163,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
 ]
 
 [[package]]
@@ -3165,7 +3174,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
 dependencies = [
  "native-tls",
- "tokio 1.29.1",
+ "tokio 1.32.0",
 ]
 
 [[package]]
@@ -3250,7 +3259,7 @@ dependencies = [
  "futures-core",
  "futures-sink",
  "pin-project-lite",
- "tokio 1.29.1",
+ "tokio 1.32.0",
  "tracing",
 ]
 
@@ -3476,8 +3485,8 @@ dependencies = [
  "log",
  "once_cell",
  "proc-macro2 1.0.66",
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
  "wasm-bindgen-shared",
 ]
 
@@ -3499,7 +3508,7 @@ version = "0.2.87"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d"
 dependencies = [
- "quote 1.0.32",
+ "quote 1.0.33",
  "wasm-bindgen-macro-support",
 ]
 
@@ -3510,8 +3519,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
 dependencies = [
  "proc-macro2 1.0.66",
- "quote 1.0.32",
- "syn 2.0.28",
+ "quote 1.0.33",
+ "syn 2.0.29",
  "wasm-bindgen-backend",
  "wasm-bindgen-shared",
 ]
@@ -3534,7 +3543,7 @@ dependencies = [
 
 [[package]]
 name = "websurfx"
-version = "0.16.4"
+version = "0.17.0"
 dependencies = [
  "actix-cors",
  "actix-files",
@@ -3556,7 +3565,7 @@ dependencies = [
  "scraper",
  "serde",
  "serde_json",
- "tokio 1.29.1",
+ "tokio 1.32.0",
 ]
 
 [[package]]
@@ -3613,9 +3622,9 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.48.1"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
 dependencies = [
  "windows_aarch64_gnullvm",
  "windows_aarch64_msvc",
@@ -3628,45 +3637,45 @@ dependencies = [
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.48.0"
+version = "0.48.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "winreg"

Cargo.toml

@@ -1,31 +1,31 @@
 [package]
 name = "websurfx"
-version = "0.16.4"
+version = "0.17.0"
 edition = "2021"
 description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
 repository = "https://github.com/neon-mmd/websurfx"
 license = "AGPL-3.0"
 
 [dependencies]
-reqwest = {version="*",features=["json"]}
-tokio = {version="*",features=["full"]}
-serde = {version="*",features=["derive"]}
-handlebars = { version = "4.3.6", features = ["dir_source"] }
-scraper = {version="*"}
+reqwest = {version="0.11.18",features=["json"]}
+tokio = {version="1.32.0",features=["full"]}
+serde = {version="1.0.183",features=["derive"]}
+handlebars = { version = "4.3.7", features = ["dir_source"] }
+scraper = {version="0.17.1"}
 actix-web = {version="4.3.1", features = ["cookies"]}
 actix-files = {version="0.6.2"}
 actix-cors = {version="0.6.4"}
-serde_json = {version="*"}
-fake-useragent = {version="*"}
+serde_json = {version="1.0.105"}
+fake-useragent = {version="0.1.3"}
 env_logger = {version="0.10.0"}
-log = {version="0.4.17"}
-rlua = {version="*"}
-redis = {version="*"}
-md5 = {version="*"}
-rand={version="*"}
-once_cell = {version="*"}
+log = {version="0.4.20"}
+rlua = {version="0.19.7"}
+redis = {version="0.23.2"}
+md5 = {version="0.7.0"}
+rand={version="0.8.5"}
+once_cell = {version="1.18.0"}
 error-stack = {version="0.3.1"}
-async-trait = {version="*"}
+async-trait = {version="0.1.73"}
 
 [dev-dependencies]
 rusty-hook = "^0.11.2"

public/templates/search.html

@@ -4,7 +4,7 @@
   <div class="results_aggregated">
     {{#if results}} {{#each results}}
     <div class="result">
-      <h1><a href="{{{this.visitingUrl}}}">{{{this.title}}}</a></h1>
+      <h1><a href="{{{this.url}}}">{{{this.title}}}</a></h1>
       <small>{{{this.url}}}</small>
       <p>{{{this.description}}}</p>
       <div class="upstream_engines">

src/bin/websurfx.rs

@@ -15,7 +15,7 @@ use websurfx::{config::parser::Config, run};
 #[actix_web::main]
 async fn main() -> std::io::Result<()> {
     // Initialize the parsed config file.
-    let config = Config::parse(true).unwrap();
+    let config = Config::parse(false).unwrap();
 
     log::info!(
         "started server on port {} and IP {}",

src/config/parser.rs

@@ -34,7 +34,7 @@ pub struct Config {
     pub aggregator: AggregatorConfig,
     pub logging: bool,
     pub debug: bool,
-    pub upstream_search_engines: Vec<String>,
+    pub upstream_search_engines: Vec<crate::engines::engine_models::EngineHandler>,
     pub request_timeout: u8,
     pub threads: u8,
 }
@@ -57,7 +57,7 @@ impl Config {
     /// # Arguments
     ///
     /// * `logging_initialized` - It takes a boolean which ensures that the logging doesn't get
-    /// initialized twice.
+    /// initialized twice. Pass false if the logger has not yet been initialized.
     ///
     /// # Error
     ///
@@ -77,22 +77,8 @@ impl Config {
             let debug: bool = globals.get::<_, bool>("debug")?;
             let logging:bool= globals.get::<_, bool>("logging")?;
 
-            // Check whether logging has not been initialized before.
-            if logging_initialized {
-                if let Ok(pkg_env_var) = std::env::var("PKG_ENV"){
-                    if pkg_env_var.to_lowercase() == "dev" {
-                        env_logger::Builder::new().filter(None, LevelFilter::Trace).init();
-                    }
-                } else {
-                    // Initializing logging middleware with level set to default or info.
-                    let mut log_level: LevelFilter = LevelFilter::Error;
-                    if logging && debug == false {
-                        log_level = LevelFilter::Info;
-                    } else if debug {
-                        log_level = LevelFilter::Debug;
-                    };
-                    env_logger::Builder::new().filter(None, log_level).init();
-                }
+            if !logging_initialized {
+                set_logging_level(debug, logging);
             }
 
             let threads: u8 = if parsed_threads == 0 {
@@ -121,12 +107,14 @@ impl Config {
                     .get::<_, HashMap<String, bool>>("upstream_search_engines")?
                     .into_iter()
                     .filter_map(|(key, value)| value.then_some(key))
+                    .filter_map(|engine| crate::engines::engine_models::EngineHandler::new(&engine))
                     .collect(),
                 request_timeout: globals.get::<_, u8>("request_timeout")?,
                 threads,
             })
         })
     }
+
     /// A helper function which returns an appropriate config file path checking if the config
     /// file exists on that path.
     ///
@@ -173,3 +161,25 @@ impl Config {
         Err("Config file not found!!".to_string().into())
     }
 }
+
+/// a helper function that sets the proper logging level
+fn set_logging_level(debug: bool, logging: bool) {
+    if let Ok(pkg_env_var) = std::env::var("PKG_ENV") {
+        if pkg_env_var.to_lowercase() == "dev" {
+            env_logger::Builder::new()
+                .filter(None, LevelFilter::Trace)
+                .init();
+            return;
+        }
+    }
+
+    // Initializing logging middleware with level set to default or info.
+    let log_level = match (debug, logging) {
+        (true, true) => LevelFilter::Debug,
+        (true, false) => LevelFilter::Debug,
+        (false, true) => LevelFilter::Info,
+        (false, false) => LevelFilter::Error,
+    };
+
+    env_logger::Builder::new().filter(None, log_level).init();
+}

src/engines/duckduckgo.rs

@@ -7,7 +7,7 @@ use std::collections::HashMap;
 use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT};
 use scraper::{Html, Selector};
 
-use crate::results::aggregation_models::RawSearchResult;
+use crate::results::aggregation_models::SearchResult;
 
 use super::engine_models::{EngineError, SearchEngine};
 
@@ -43,7 +43,7 @@ impl SearchEngine for DuckDuckGo {
         page: u32,
         user_agent: String,
         request_timeout: u8,
-    ) -> Result<HashMap<String, RawSearchResult>, EngineError> {
+    ) -> Result<HashMap<String, SearchResult>, EngineError> {
         // Page number can be missing or empty string and so appropriate handling is required
         // so that upstream server recieves valid page number.
         let url: String = match page {
@@ -120,7 +120,7 @@ impl SearchEngine for DuckDuckGo {
         Ok(document
             .select(&results)
             .map(|result| {
-                RawSearchResult::new(
+                SearchResult::new(
                     result
                         .select(&result_title)
                         .next()
@@ -147,7 +147,7 @@ impl SearchEngine for DuckDuckGo {
                     vec!["duckduckgo".to_string()],
                 )
             })
-            .map(|search_result| (search_result.visiting_url.clone(), search_result))
+            .map(|search_result| (search_result.url.clone(), search_result))
             .collect())
     }
 }

src/engines/engine_models.rs

@@ -1,7 +1,7 @@
 //! This module provides the error enum to handle different errors associated while requesting data from
 //! the upstream search engines with the search query provided by the user.
 
-use crate::results::aggregation_models::RawSearchResult;
+use crate::results::aggregation_models::SearchResult;
 use error_stack::{IntoReport, Result, ResultExt};
 use std::{collections::HashMap, fmt, time::Duration};
 
@@ -43,9 +43,9 @@ impl fmt::Display for EngineError {
 
 impl error_stack::Context for EngineError {}
 
-/// A trait to define common behaviour for all search engines.
+/// A trait to define common behavior for all search engines.
 #[async_trait::async_trait]
-pub trait SearchEngine {
+pub trait SearchEngine: Sync + Send {
     async fn fetch_html_from_upstream(
         &self,
         url: String,
@@ -56,7 +56,7 @@ pub trait SearchEngine {
         Ok(reqwest::Client::new()
             .get(url)
             .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server
-            .headers(header_map) // add spoofed headers to emulate human behaviour
+            .headers(header_map) // add spoofed headers to emulate human behavior
             .send()
             .await
             .into_report()
@@ -73,5 +73,37 @@ pub trait SearchEngine {
         page: u32,
         user_agent: String,
         request_timeout: u8,
-    ) -> Result<HashMap<String, RawSearchResult>, EngineError>;
+    ) -> Result<HashMap<String, SearchResult>, EngineError>;
+}
+
+pub struct EngineHandler {
+    engine: Box<dyn SearchEngine>,
+    name: &'static str,
+}
+
+impl Clone for EngineHandler {
+    fn clone(&self) -> Self {
+        Self::new(self.name).unwrap()
+    }
+}
+
+impl EngineHandler {
+    /// parses an engine name into an engine handler, returns none if the engine is unknown
+    pub fn new(engine_name: &str) -> Option<Self> {
+        let engine: (&'static str, Box<dyn SearchEngine>) =
+            match engine_name.to_lowercase().as_str() {
+                "duckduckgo" => ("duckduckgo", Box::new(super::duckduckgo::DuckDuckGo)),
+                "searx" => ("searx", Box::new(super::searx::Searx)),
+                _ => return None,
+            };
+
+        Some(Self {
+            engine: engine.1,
+            name: engine.0,
+        })
+    }
+
+    pub fn into_name_engine(self) -> (&'static str, Box<dyn SearchEngine>) {
+        (self.name, self.engine)
+    }
 }

src/engines/searx.rs

@@ -6,7 +6,7 @@ use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT};
 use scraper::{Html, Selector};
 use std::collections::HashMap;
 
-use crate::results::aggregation_models::RawSearchResult;
+use crate::results::aggregation_models::SearchResult;
 
 use super::engine_models::{EngineError, SearchEngine};
 use error_stack::{IntoReport, Report, Result, ResultExt};
@@ -42,7 +42,7 @@ impl SearchEngine for Searx {
         page: u32,
         user_agent: String,
         request_timeout: u8,
-    ) -> Result<HashMap<String, RawSearchResult>, EngineError> {
+    ) -> Result<HashMap<String, SearchResult>, EngineError> {
         // Page number can be missing or empty string and so appropriate handling is required
         // so that upstream server recieves valid page number.
         let url: String = match page {
@@ -111,7 +111,7 @@ impl SearchEngine for Searx {
         Ok(document
             .select(&results)
             .map(|result| {
-                RawSearchResult::new(
+                SearchResult::new(
                     result
                         .select(&result_title)
                         .next()
@@ -137,7 +137,7 @@ impl SearchEngine for Searx {
                     vec!["searx".to_string()],
                 )
             })
-            .map(|search_result| (search_result.visiting_url.clone(), search_result))
+            .map(|search_result| (search_result.url.clone(), search_result))
             .collect())
     }
 }

src/results/aggregation_models.rs

@@ -5,97 +5,44 @@ use serde::{Deserialize, Serialize};
 
 use crate::{config::parser_models::Style, engines::engine_models::EngineError};
 
-/// A named struct to store, serialize and deserializes the individual search result from all the
-/// scraped and aggregated search results from the upstream search engines.
+/// A named struct to store the raw scraped search results scraped search results from the
+/// upstream search engines before aggregating it.It derives the Clone trait which is needed
+/// to write idiomatic rust using `Iterators`.
 ///
 /// # Fields
 ///
 /// * `title` - The title of the search result.
-/// * `visiting_url` - The url which is accessed when clicked on it (href url in html in simple
-/// words).
-/// * `url` - The url to be displayed below the search result title in html.
+/// * `url` - The url which is accessed when clicked on it
+/// (href url in html in simple words).
 /// * `description` - The description of the search result.
 /// * `engine` - The names of the upstream engines from which this results were provided.
-#[derive(Serialize, Deserialize)]
+#[derive(Clone, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {
     pub title: String,
-    pub visiting_url: String,
     pub url: String,
     pub description: String,
     pub engine: Vec<String>,
 }
 
 impl SearchResult {
-    /// Constructs a new `SearchResult` with the given arguments needed for the struct.
+    /// Constructs a new `RawSearchResult` with the given arguments needed for the struct.
     ///
     /// # Arguments
     ///
     /// * `title` - The title of the search result.
-    /// * `visiting_url` - The url which is accessed when clicked on it
+    /// * `url` - The url which is accessed when clicked on it
     /// (href url in html in simple words).
-    /// * `url` - The url to be displayed below the search result title in html.
     /// * `description` - The description of the search result.
     /// * `engine` - The names of the upstream engines from which this results were provided.
-    pub fn new(
-        title: String,
-        visiting_url: String,
-        url: String,
-        description: String,
-        engine: Vec<String>,
-    ) -> Self {
+    pub fn new(title: String, url: String, description: String, engine: Vec<String>) -> Self {
         SearchResult {
             title,
-            visiting_url,
             url,
             description,
             engine,
         }
     }
-}
-
-/// A named struct to store the raw scraped search results scraped search results from the
-/// upstream search engines before aggregating it.It derives the Clone trait which is needed
-/// to write idiomatic rust using `Iterators`.
-///
-/// # Fields
-///
-/// * `title` - The title of the search result.
-/// * `visiting_url` - The url which is accessed when clicked on it
-/// (href url in html in simple words).
-/// * `description` - The description of the search result.
-/// * `engine` - The names of the upstream engines from which this results were provided.
-#[derive(Clone)]
-pub struct RawSearchResult {
-    pub title: String,
-    pub visiting_url: String,
-    pub description: String,
-    pub engine: Vec<String>,
-}
-
-impl RawSearchResult {
-    /// Constructs a new `RawSearchResult` with the given arguments needed for the struct.
-    ///
-    /// # Arguments
-    ///
-    /// * `title` - The title of the search result.
-    /// * `visiting_url` - The url which is accessed when clicked on it
-    /// (href url in html in simple words).
-    /// * `description` - The description of the search result.
-    /// * `engine` - The names of the upstream engines from which this results were provided.
-    pub fn new(
-        title: String,
-        visiting_url: String,
-        description: String,
-        engine: Vec<String>,
-    ) -> Self {
-        RawSearchResult {
-            title,
-            visiting_url,
-            description,
-            engine,
-        }
-    }
 
     /// A function which adds the engine name provided as a string into a vector of strings.
     ///

src/results/aggregator.rs

@@ -8,18 +8,14 @@ use rand::Rng;
 use tokio::task::JoinHandle;
 
 use super::{
-    aggregation_models::{EngineErrorInfo, RawSearchResult, SearchResult, SearchResults},
+    aggregation_models::{EngineErrorInfo, SearchResult, SearchResults},
     user_agent::random_user_agent,
 };
 
-use crate::engines::{
-    duckduckgo,
-    engine_models::{EngineError, SearchEngine},
-    searx,
-};
+use crate::engines::engine_models::{EngineError, EngineHandler};
 
 /// Aliases for long type annotations
-type FutureVec = Vec<JoinHandle<Result<HashMap<String, RawSearchResult>, Report<EngineError>>>>;
+type FutureVec = Vec<JoinHandle<Result<HashMap<String, SearchResult>, Report<EngineError>>>>;
 
 /// The function aggregates the scraped results from the user-selected upstream search engines.
 /// These engines can be chosen either from the user interface (UI) or from the configuration file.
@@ -64,139 +60,93 @@ pub async fn aggregate(
     page: u32,
     random_delay: bool,
     debug: bool,
-    upstream_search_engines: Vec<String>,
+    upstream_search_engines: Vec<EngineHandler>,
     request_timeout: u8,
 ) -> Result<SearchResults, Box<dyn std::error::Error>> {
     let user_agent: String = random_user_agent();
-    let mut result_map: HashMap<String, RawSearchResult> = HashMap::new();
 
     // Add a random delay before making the request.
     if random_delay || !debug {
         let mut rng = rand::thread_rng();
         let delay_secs = rng.gen_range(1..10);
-        std::thread::sleep(Duration::from_secs(delay_secs));
+        tokio::time::sleep(Duration::from_secs(delay_secs)).await;
     }
 
-    // fetch results from upstream search engines simultaneously/concurrently.
-    let search_engines: Vec<Box<dyn SearchEngine + Send + Sync>> = upstream_search_engines
-        .iter()
-        .map(|engine| match engine.to_lowercase().as_str() {
-            "duckduckgo" => Box::new(duckduckgo::DuckDuckGo) as Box<dyn SearchEngine + Send + Sync>,
-            "searx" => Box::new(searx::Searx) as Box<dyn SearchEngine + Send + Sync>,
-            &_ => panic!("Config Error: Incorrect config file option provided"),
-        })
-        .collect();
-
-    let task_capacity: usize = search_engines.len();
-
-    let tasks: FutureVec = search_engines
-        .into_iter()
-        .map(|search_engine| {
-            let query: String = query.clone();
-            let user_agent: String = user_agent.clone();
-            tokio::spawn(async move {
-                search_engine
-                    .results(query, page, user_agent.clone(), request_timeout)
-                    .await
-            })
-        })
-        .collect();
-
-    let mut outputs = Vec::with_capacity(task_capacity);
+    let mut names: Vec<&str> = vec![];
+
+    // create tasks for upstream result fetching
+    let mut tasks: FutureVec = FutureVec::new();
+
+    for engine_handler in upstream_search_engines {
+        let (name, search_engine) = engine_handler.into_name_engine();
+        names.push(name);
+        let query: String = query.clone();
+        let user_agent: String = user_agent.clone();
+        tasks.push(tokio::spawn(async move {
+            search_engine
+                .results(query, page, user_agent.clone(), request_timeout)
+                .await
+        }));
+    }
+
+    // get upstream responses
+    let mut responses = Vec::with_capacity(tasks.len());
 
     for task in tasks {
         if let Ok(result) = task.await {
-            outputs.push(result)
+            responses.push(result)
         }
     }
 
+    // aggregate search results, removing duplicates and handling errors the upstream engines returned
+    let mut result_map: HashMap<String, SearchResult> = HashMap::new();
     let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new();
 
-    // The code block `outputs.iter()` determines whether it is the first time the code is being run.
-    // It does this by checking the initial flag. If it is the first time, the code selects the first
-    // engine from which results are fetched and adds or extends them into the `result_map`. If the
-    // initially selected engine fails, the code automatically selects another engine to map or extend
-    // into the `result_map`. On the other hand, if an engine selected for the first time successfully
-    // fetches results and maps them into the `result_map`, the initial flag is set to false. Subsequently,
-    // the code iterates through the remaining engines one by one. It compares the fetched results from each
-    // engine with the results already present in the `result_map` to identify any duplicates. If duplicate
-    // results are found, the code groups them together with the name of the engine from which they were
-    // fetched, and automatically removes the duplicate results from the newly fetched data.
-    //
-    // Additionally, the code handles errors returned by the engines. It keeps track of which engines
-    // encountered errors and stores this information in a vector of structures called `EngineErrorInfo`.
-    // Each structure in this vector contains the name of the engine and the type of error it returned.
-    // These structures will later be added to the final `SearchResults` structure. The `SearchResults`
-    // structure is used to display an error box in the UI containing the relevant information from
-    // the `EngineErrorInfo` structure.
-    //
-    // In summary, this code block manages the selection of engines, handling of duplicate results, and tracking
-    // of errors in order to populate the `result_map` and provide informative feedback to the user through the
-    // `SearchResults` structure.
-    let mut initial: bool = true;
-    let mut counter: usize = 0;
-    outputs.iter().for_each(|results| {
-        if initial {
-            match results {
-                Ok(result) => {
-                    result_map.extend(result.clone());
-                    counter += 1;
-                    initial = false
+    let mut handle_error = |error: Report<EngineError>, engine_name: String| {
+        log::error!("Engine Error: {:?}", error);
+        engine_errors_info.push(EngineErrorInfo::new(
+            error.downcast_ref::<EngineError>().unwrap(),
+            engine_name.to_string(),
+        ));
+    };
+
+    for _ in 0..responses.len() {
+        let response = responses.pop().unwrap();
+        let engine = names.pop().unwrap().to_string();
+
+        if result_map.is_empty() {
+            match response {
+                Ok(results) => {
+                    result_map = results.clone();
                 }
-                Err(error_type) => {
-                    log::error!("Engine Error: {:?}", error_type);
-                    engine_errors_info.push(EngineErrorInfo::new(
-                        error_type.downcast_ref::<EngineError>().unwrap(),
-                        upstream_search_engines[counter].clone(),
-                    ));
-                    counter += 1
+                Err(error) => {
+                    handle_error(error, engine);
                 }
             }
-        } else {
-            match results {
-                Ok(result) => {
-                    result.clone().into_iter().for_each(|(key, value)| {
-                        result_map
-                            .entry(key)
-                            .and_modify(|result| {
-                                result.add_engines(value.clone().engine());
-                            })
-                            .or_insert_with(|| -> RawSearchResult {
-                                RawSearchResult::new(
-                                    value.title.clone(),
-                                    value.visiting_url.clone(),
-                                    value.description.clone(),
-                                    value.engine.clone(),
-                                )
-                            });
-                    });
-                    counter += 1
-                }
-                Err(error_type) => {
-                    log::error!("Engine Error: {:?}", error_type);
-                    engine_errors_info.push(EngineErrorInfo::new(
-                        error_type.downcast_ref::<EngineError>().unwrap(),
-                        upstream_search_engines[counter].clone(),
-                    ));
-                    counter += 1
-                }
+            continue;
+        }
+
+        match response {
+            Ok(result) => {
+                result.into_iter().for_each(|(key, value)| {
+                    result_map
+                        .entry(key)
+                        .and_modify(|result| {
+                            result.add_engines(engine.clone());
+                        })
+                        .or_insert_with(|| -> SearchResult { value });
+                });
+            }
+            Err(error) => {
+                handle_error(error, engine);
             }
         }
-    });
+    }
+
+    let results = result_map.into_values().collect();
 
     Ok(SearchResults::new(
-        result_map
-            .into_iter()
-            .map(|(key, value)| {
-                SearchResult::new(
-                    value.title,
-                    value.visiting_url,
-                    key,
-                    value.description,
-                    value.engine,
-                )
-            })
-            .collect(),
+        results,
         query.to_string(),
         engine_errors_info,
     ))

src/server/routes.rs

@@ -7,12 +7,14 @@ use std::fs::read_to_string;
 use crate::{
     cache::cacher::RedisCache,
     config::parser::Config,
+    engines::engine_models::EngineHandler,
     handler::public_paths::public_path,
     results::{aggregation_models::SearchResults, aggregator::aggregate},
 };
 use actix_web::{get, web, HttpRequest, HttpResponse};
 use handlebars::Handlebars;
 use serde::Deserialize;
+use tokio::join;
 
 /// A named struct which deserializes all the user provided search parameters and stores them.
 ///
@@ -96,15 +98,49 @@ pub async fn search(
             }
             let page = match &params.page {
                 Some(page) => *page,
-                None => 0,
+                None => 1,
             };
 
-            let url = format!(
-                "http://{}:{}/search?q={}&page={}",
-                config.binding_ip, config.port, query, page
+            let (_, results, _) = join!(
+                results(
+                    format!(
+                        "http://{}:{}/search?q={}&page={}",
+                        config.binding_ip,
+                        config.port,
+                        query,
+                        page - 1
+                    ),
+                    &config,
+                    query.to_string(),
+                    page - 1,
+                    req.clone(),
+                ),
+                results(
+                    format!(
+                        "http://{}:{}/search?q={}&page={}",
+                        config.binding_ip, config.port, query, page
+                    ),
+                    &config,
+                    query.to_string(),
+                    page,
+                    req.clone(),
+                ),
+                results(
+                    format!(
+                        "http://{}:{}/search?q={}&page={}",
+                        config.binding_ip,
+                        config.port,
+                        query,
+                        page + 1
+                    ),
+                    &config,
+                    query.to_string(),
+                    page + 1,
+                    req.clone(),
+                )
             );
-            let results_json = results(url, &config, query.to_string(), page, req).await?;
-            let page_content: String = hbs.render("search", &results_json)?;
+
+            let page_content: String = hbs.render("search", &results?)?;
             Ok(HttpResponse::Ok().body(page_content))
         }
         None => Ok(HttpResponse::Found()
@@ -140,12 +176,19 @@ async fn results(
             {
                 Some(cookie_value) => {
                     let cookie_value: Cookie = serde_json::from_str(cookie_value.name_value().1)?;
+
+                    let engines = cookie_value
+                        .engines
+                        .iter()
+                        .filter_map(|name| EngineHandler::new(name))
+                        .collect();
+
                     aggregate(
                         query,
                         page,
                         config.aggregator.random_delay,
                         config.debug,
-                        cookie_value.engines,
+                        engines,
                         config.request_timeout,
                     )
                     .await?

tests/index.rs

@@ -8,7 +8,7 @@ fn spawn_app() -> String {
     // Binding to port 0 will trigger the OS to assign a port for us.
     let listener = TcpListener::bind("127.0.0.1:0").expect("Failed to bind random port");
     let port = listener.local_addr().unwrap().port();
-    let config = Config::parse(true).unwrap();
+    let config = Config::parse(false).unwrap();
     let server = run(listener, config).expect("Failed to bind address");
 
     tokio::spawn(server);
@@ -36,7 +36,7 @@ async fn test_index() {
     assert_eq!(res.status(), 200);
 
     let handlebars = handlebars();
-    let config = Config::parse(false).unwrap();
+    let config = Config::parse(true).unwrap();
     let template = handlebars.render("index", &config.style).unwrap();
     assert_eq!(res.text().await.unwrap(), template);
 }