neon_arch commited on
Commit
ca4447f
2 Parent(s): 485a5a1 7a64454

Merge branch 'rolling' into reorganize-code-and-restructure-the-codebase

Browse files
.gitignore CHANGED
@@ -4,3 +4,4 @@ package-lock.json
4
  dump.rdb
5
  .vscode
6
  megalinter-reports/
 
 
4
  dump.rdb
5
  .vscode
6
  megalinter-reports/
7
+ dhat-heap.json
Cargo.lock CHANGED
@@ -9,7 +9,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
9
  checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8"
10
  dependencies = [
11
  "bitflags 1.3.2",
12
- "bytes 1.4.0",
13
  "futures-core",
14
  "futures-sink",
15
  "memchr",
@@ -46,7 +46,7 @@ dependencies = [
46
  "actix-web",
47
  "askama_escape",
48
  "bitflags 1.3.2",
49
- "bytes 1.4.0",
50
  "derive_more",
51
  "futures-core",
52
  "http-range",
@@ -57,6 +57,18 @@ dependencies = [
57
  "pin-project-lite",
58
  ]
59
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  [[package]]
61
  name = "actix-http"
62
  version = "3.4.0"
@@ -68,10 +80,10 @@ dependencies = [
68
  "actix-service",
69
  "actix-utils",
70
  "ahash",
71
- "base64 0.21.3",
72
  "bitflags 2.4.0",
73
  "brotli",
74
- "bytes 1.4.0",
75
  "bytestring",
76
  "derive_more",
77
  "encoding_rs",
@@ -103,7 +115,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
103
  checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
104
  dependencies = [
105
  "quote 1.0.33",
106
- "syn 2.0.31",
107
  ]
108
 
109
  [[package]]
@@ -183,7 +195,7 @@ dependencies = [
183
  "actix-utils",
184
  "actix-web-codegen",
185
  "ahash",
186
- "bytes 1.4.0",
187
  "bytestring",
188
  "cfg-if 1.0.0",
189
  "cookie 0.16.2",
@@ -216,7 +228,7 @@ dependencies = [
216
  "actix-router",
217
  "proc-macro2 1.0.66",
218
  "quote 1.0.33",
219
- "syn 2.0.31",
220
  ]
221
 
222
  [[package]]
@@ -278,9 +290,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
278
 
279
  [[package]]
280
  name = "anstyle"
281
- version = "1.0.2"
282
  source = "registry+https://github.com/rust-lang/crates.io-index"
283
- checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea"
284
 
285
  [[package]]
286
  name = "anyhow"
@@ -288,12 +300,24 @@ version = "1.0.75"
288
  source = "registry+https://github.com/rust-lang/crates.io-index"
289
  checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
290
 
 
 
 
 
 
 
291
  [[package]]
292
  name = "askama_escape"
293
  version = "0.10.3"
294
  source = "registry+https://github.com/rust-lang/crates.io-index"
295
  checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
296
 
 
 
 
 
 
 
297
  [[package]]
298
  name = "async-trait"
299
  version = "0.1.73"
@@ -302,7 +326,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0"
302
  dependencies = [
303
  "proc-macro2 1.0.66",
304
  "quote 1.0.33",
305
- "syn 2.0.31",
306
  ]
307
 
308
  [[package]]
@@ -346,9 +370,9 @@ dependencies = [
346
 
347
  [[package]]
348
  name = "base64"
349
- version = "0.21.3"
350
  source = "registry+https://github.com/rust-lang/crates.io-index"
351
- checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53"
352
 
353
  [[package]]
354
  name = "bit-set"
@@ -441,9 +465,9 @@ dependencies = [
441
 
442
  [[package]]
443
  name = "bytes"
444
- version = "1.4.0"
445
  source = "registry+https://github.com/rust-lang/crates.io-index"
446
- checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
447
 
448
  [[package]]
449
  name = "bytestring"
@@ -451,7 +475,7 @@ version = "1.3.0"
451
  source = "registry+https://github.com/rust-lang/crates.io-index"
452
  checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae"
453
  dependencies = [
454
- "bytes 1.4.0",
455
  ]
456
 
457
  [[package]]
@@ -558,8 +582,12 @@ version = "4.6.6"
558
  source = "registry+https://github.com/rust-lang/crates.io-index"
559
  checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
560
  dependencies = [
561
- "bytes 1.4.0",
 
562
  "memchr",
 
 
 
563
  ]
564
 
565
  [[package]]
@@ -574,7 +602,7 @@ version = "0.12.0"
574
  source = "registry+https://github.com/rust-lang/crates.io-index"
575
  checksum = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
576
  dependencies = [
577
- "time 0.1.45",
578
  "url 1.7.2",
579
  ]
580
 
@@ -602,7 +630,7 @@ dependencies = [
602
  "publicsuffix",
603
  "serde",
604
  "serde_json",
605
- "time 0.1.45",
606
  "try_from",
607
  "url 1.7.2",
608
  ]
@@ -798,7 +826,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
798
  checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
799
  dependencies = [
800
  "quote 1.0.33",
801
- "syn 2.0.31",
 
 
 
 
 
 
 
 
 
 
 
 
 
802
  ]
803
 
804
  [[package]]
@@ -820,6 +861,22 @@ dependencies = [
820
  "syn 1.0.109",
821
  ]
822
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
823
  [[package]]
824
  name = "digest"
825
  version = "0.10.7"
@@ -918,9 +975,9 @@ dependencies = [
918
 
919
  [[package]]
920
  name = "error-stack"
921
- version = "0.4.0"
922
  source = "registry+https://github.com/rust-lang/crates.io-index"
923
- checksum = "e6a37ef405b504fc3b87a24fa52906d98cdd1a7d4e5ef2b49f0d5fead138fced"
924
  dependencies = [
925
  "anyhow",
926
  "rustc_version 0.4.0",
@@ -1049,6 +1106,21 @@ version = "0.1.31"
1049
  source = "registry+https://github.com/rust-lang/crates.io-index"
1050
  checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
1051
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1052
  [[package]]
1053
  name = "futures-channel"
1054
  version = "0.3.28"
@@ -1056,6 +1128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1056
  checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
1057
  dependencies = [
1058
  "futures-core",
 
1059
  ]
1060
 
1061
  [[package]]
@@ -1070,10 +1143,38 @@ version = "0.1.8"
1070
  source = "registry+https://github.com/rust-lang/crates.io-index"
1071
  checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
1072
  dependencies = [
1073
- "futures",
1074
  "num_cpus",
1075
  ]
1076
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1077
  [[package]]
1078
  name = "futures-sink"
1079
  version = "0.3.28"
@@ -1086,16 +1187,28 @@ version = "0.3.28"
1086
  source = "registry+https://github.com/rust-lang/crates.io-index"
1087
  checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
1088
 
 
 
 
 
 
 
1089
  [[package]]
1090
  name = "futures-util"
1091
  version = "0.3.28"
1092
  source = "registry+https://github.com/rust-lang/crates.io-index"
1093
  checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
1094
  dependencies = [
 
1095
  "futures-core",
 
 
 
1096
  "futures-task",
 
1097
  "pin-project-lite",
1098
  "pin-utils",
 
1099
  ]
1100
 
1101
  [[package]]
@@ -1143,6 +1256,24 @@ version = "0.28.0"
1143
  source = "registry+https://github.com/rust-lang/crates.io-index"
1144
  checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
1145
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1146
  [[package]]
1147
  name = "h2"
1148
  version = "0.1.26"
@@ -1152,7 +1283,7 @@ dependencies = [
1152
  "byteorder",
1153
  "bytes 0.4.12",
1154
  "fnv",
1155
- "futures",
1156
  "http 0.1.21",
1157
  "indexmap",
1158
  "log",
@@ -1167,7 +1298,7 @@ version = "0.3.21"
1167
  source = "registry+https://github.com/rust-lang/crates.io-index"
1168
  checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833"
1169
  dependencies = [
1170
- "bytes 1.4.0",
1171
  "fnv",
1172
  "futures-core",
1173
  "futures-sink",
@@ -1207,6 +1338,12 @@ version = "0.12.3"
1207
  source = "registry+https://github.com/rust-lang/crates.io-index"
1208
  checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
1209
 
 
 
 
 
 
 
1210
  [[package]]
1211
  name = "hermit-abi"
1212
  version = "0.3.2"
@@ -1258,7 +1395,7 @@ version = "0.2.9"
1258
  source = "registry+https://github.com/rust-lang/crates.io-index"
1259
  checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
1260
  dependencies = [
1261
- "bytes 1.4.0",
1262
  "fnv",
1263
  "itoa 1.0.9",
1264
  ]
@@ -1270,7 +1407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1270
  checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
1271
  dependencies = [
1272
  "bytes 0.4.12",
1273
- "futures",
1274
  "http 0.1.21",
1275
  "tokio-buf",
1276
  ]
@@ -1281,7 +1418,7 @@ version = "0.4.5"
1281
  source = "registry+https://github.com/rust-lang/crates.io-index"
1282
  checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
1283
  dependencies = [
1284
- "bytes 1.4.0",
1285
  "http 0.2.9",
1286
  "pin-project-lite",
1287
  ]
@@ -1317,7 +1454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1317
  checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52"
1318
  dependencies = [
1319
  "bytes 0.4.12",
1320
- "futures",
1321
  "futures-cpupool",
1322
  "h2 0.1.26",
1323
  "http 0.1.21",
@@ -1328,7 +1465,7 @@ dependencies = [
1328
  "log",
1329
  "net2",
1330
  "rustc_version 0.2.3",
1331
- "time 0.1.45",
1332
  "tokio 0.1.22",
1333
  "tokio-buf",
1334
  "tokio-executor",
@@ -1346,7 +1483,7 @@ version = "0.14.27"
1346
  source = "registry+https://github.com/rust-lang/crates.io-index"
1347
  checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
1348
  dependencies = [
1349
- "bytes 1.4.0",
1350
  "futures-channel",
1351
  "futures-core",
1352
  "futures-util",
@@ -1371,7 +1508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1371
  checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f"
1372
  dependencies = [
1373
  "bytes 0.4.12",
1374
- "futures",
1375
  "hyper 0.12.36",
1376
  "native-tls",
1377
  "tokio-io",
@@ -1383,7 +1520,7 @@ version = "0.5.0"
1383
  source = "registry+https://github.com/rust-lang/crates.io-index"
1384
  checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
1385
  dependencies = [
1386
- "bytes 1.4.0",
1387
  "hyper 0.14.27",
1388
  "native-tls",
1389
  "tokio 1.32.0",
@@ -1429,7 +1566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
1429
  checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
1430
  dependencies = [
1431
  "autocfg 1.1.0",
1432
- "hashbrown",
1433
  ]
1434
 
1435
  [[package]]
@@ -1525,11 +1662,21 @@ version = "0.2.147"
1525
  source = "registry+https://github.com/rust-lang/crates.io-index"
1526
  checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
1527
 
 
 
 
 
 
 
 
 
 
 
1528
  [[package]]
1529
  name = "linux-raw-sys"
1530
- version = "0.4.5"
1531
  source = "registry+https://github.com/rust-lang/crates.io-index"
1532
- checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503"
1533
 
1534
  [[package]]
1535
  name = "local-channel"
@@ -1580,6 +1727,15 @@ version = "0.1.1"
1580
  source = "registry+https://github.com/rust-lang/crates.io-index"
1581
  checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
1582
 
 
 
 
 
 
 
 
 
 
1583
  [[package]]
1584
  name = "markup5ever"
1585
  version = "0.8.1"
@@ -1653,6 +1809,15 @@ dependencies = [
1653
  "autocfg 1.1.0",
1654
  ]
1655
 
 
 
 
 
 
 
 
 
 
1656
  [[package]]
1657
  name = "mime"
1658
  version = "0.3.17"
@@ -1678,6 +1843,16 @@ dependencies = [
1678
  "adler",
1679
  ]
1680
 
 
 
 
 
 
 
 
 
 
 
1681
  [[package]]
1682
  name = "mio"
1683
  version = "0.6.23"
@@ -1721,6 +1896,20 @@ dependencies = [
1721
  "ws2_32-sys",
1722
  ]
1723
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1724
  [[package]]
1725
  name = "native-tls"
1726
  version = "0.2.11"
@@ -1762,6 +1951,18 @@ version = "0.5.0"
1762
  source = "registry+https://github.com/rust-lang/crates.io-index"
1763
  checksum = "ab250442c86f1850815b5d268639dff018c0627022bc1940eb2d642ca1ce12f0"
1764
 
 
 
 
 
 
 
 
 
 
 
 
 
1765
  [[package]]
1766
  name = "num-traits"
1767
  version = "0.2.16"
@@ -1825,7 +2026,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
1825
  dependencies = [
1826
  "proc-macro2 1.0.66",
1827
  "quote 1.0.33",
1828
- "syn 2.0.31",
1829
  ]
1830
 
1831
  [[package]]
@@ -1836,9 +2037,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
1836
 
1837
  [[package]]
1838
  name = "openssl-sys"
1839
- version = "0.9.92"
1840
  source = "registry+https://github.com/rust-lang/crates.io-index"
1841
- checksum = "db7e971c2c2bba161b2d2fdf37080177eff520b3bc044787c7f1f5f9e78d869b"
1842
  dependencies = [
1843
  "cc",
1844
  "libc",
@@ -1944,7 +2145,7 @@ dependencies = [
1944
  "pest_meta",
1945
  "proc-macro2 1.0.66",
1946
  "quote 1.0.33",
1947
- "syn 2.0.31",
1948
  ]
1949
 
1950
  [[package]]
@@ -2046,7 +2247,7 @@ dependencies = [
2046
  "phf_shared 0.11.2",
2047
  "proc-macro2 1.0.66",
2048
  "quote 1.0.33",
2049
- "syn 2.0.31",
2050
  ]
2051
 
2052
  [[package]]
@@ -2076,6 +2277,26 @@ dependencies = [
2076
  "siphasher 0.3.11",
2077
  ]
2078
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2079
  [[package]]
2080
  name = "pin-project-lite"
2081
  version = "0.2.13"
@@ -2162,6 +2383,22 @@ dependencies = [
2162
  "url 2.4.1",
2163
  ]
2164
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2165
  [[package]]
2166
  name = "quote"
2167
  version = "0.6.13"
@@ -2316,6 +2553,15 @@ dependencies = [
2316
  "rand_core 0.3.1",
2317
  ]
2318
 
 
 
 
 
 
 
 
 
 
2319
  [[package]]
2320
  name = "rayon"
2321
  version = "1.7.0"
@@ -2353,12 +2599,21 @@ version = "0.23.3"
2353
  source = "registry+https://github.com/rust-lang/crates.io-index"
2354
  checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
2355
  dependencies = [
 
 
 
2356
  "combine",
 
 
2357
  "itoa 1.0.9",
2358
  "percent-encoding 2.3.0",
 
2359
  "ryu",
2360
  "sha1_smol",
2361
  "socket2 0.4.9",
 
 
 
2362
  "url 2.4.1",
2363
  ]
2364
 
@@ -2418,7 +2673,7 @@ dependencies = [
2418
  "cookie_store",
2419
  "encoding_rs",
2420
  "flate2",
2421
- "futures",
2422
  "http 0.1.21",
2423
  "hyper 0.12.36",
2424
  "hyper-tls 0.3.2",
@@ -2429,7 +2684,7 @@ dependencies = [
2429
  "serde",
2430
  "serde_json",
2431
  "serde_urlencoded 0.5.5",
2432
- "time 0.1.45",
2433
  "tokio 0.1.22",
2434
  "tokio-executor",
2435
  "tokio-io",
@@ -2446,8 +2701,8 @@ version = "0.11.20"
2446
  source = "registry+https://github.com/rust-lang/crates.io-index"
2447
  checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
2448
  dependencies = [
2449
- "base64 0.21.3",
2450
- "bytes 1.4.0",
2451
  "encoding_rs",
2452
  "futures-core",
2453
  "futures-util",
@@ -2477,36 +2732,18 @@ dependencies = [
2477
  "winreg 0.50.0",
2478
  ]
2479
 
2480
- [[package]]
2481
- name = "rlua"
2482
- version = "0.19.7"
2483
- source = "registry+https://github.com/rust-lang/crates.io-index"
2484
- checksum = "5d33e5ba15c3d43178f283ed5863d4531e292fc0e56fb773f3bea45f18e3a42a"
2485
- dependencies = [
2486
- "bitflags 1.3.2",
2487
- "bstr",
2488
- "libc",
2489
- "num-traits",
2490
- "rlua-lua54-sys",
2491
- ]
2492
-
2493
- [[package]]
2494
- name = "rlua-lua54-sys"
2495
- version = "0.1.6"
2496
- source = "registry+https://github.com/rust-lang/crates.io-index"
2497
- checksum = "7aafabafe1895cb4a2be81a56d7ff3d46bf4b5d2f9cfdbea2ed404cdabe96474"
2498
- dependencies = [
2499
- "cc",
2500
- "libc",
2501
- "pkg-config",
2502
- ]
2503
-
2504
  [[package]]
2505
  name = "rustc-demangle"
2506
  version = "0.1.23"
2507
  source = "registry+https://github.com/rust-lang/crates.io-index"
2508
  checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
2509
 
 
 
 
 
 
 
2510
  [[package]]
2511
  name = "rustc_version"
2512
  version = "0.2.3"
@@ -2527,9 +2764,9 @@ dependencies = [
2527
 
2528
  [[package]]
2529
  name = "rustix"
2530
- version = "0.38.11"
2531
  source = "registry+https://github.com/rust-lang/crates.io-index"
2532
- checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453"
2533
  dependencies = [
2534
  "bitflags 2.4.0",
2535
  "errno",
@@ -2687,14 +2924,14 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
2687
  dependencies = [
2688
  "proc-macro2 1.0.66",
2689
  "quote 1.0.33",
2690
- "syn 2.0.31",
2691
  ]
2692
 
2693
  [[package]]
2694
  name = "serde_json"
2695
- version = "1.0.105"
2696
  source = "registry+https://github.com/rust-lang/crates.io-index"
2697
- checksum = "693151e1ac27563d6dbcec9dee9fbd5da8539b20fa14ad3752b2e6d363ace360"
2698
  dependencies = [
2699
  "itoa 1.0.9",
2700
  "ryu",
@@ -2806,6 +3043,9 @@ name = "smallvec"
2806
  version = "1.11.0"
2807
  source = "registry+https://github.com/rust-lang/crates.io-index"
2808
  checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
 
 
 
2809
 
2810
  [[package]]
2811
  name = "socket2"
@@ -2926,9 +3166,9 @@ dependencies = [
2926
 
2927
  [[package]]
2928
  name = "syn"
2929
- version = "2.0.31"
2930
  source = "registry+https://github.com/rust-lang/crates.io-index"
2931
- checksum = "718fa2415bcb8d8bd775917a1bf12a7931b6dfa890753378538118181e0cb398"
2932
  dependencies = [
2933
  "proc-macro2 1.0.66",
2934
  "quote 1.0.33",
@@ -2947,6 +3187,16 @@ dependencies = [
2947
  "unicode-xid 0.2.4",
2948
  ]
2949
 
 
 
 
 
 
 
 
 
 
 
2950
  [[package]]
2951
  name = "tempfile"
2952
  version = "3.8.0"
@@ -2997,17 +3247,22 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35"
2997
  dependencies = [
2998
  "proc-macro2 1.0.66",
2999
  "quote 1.0.33",
3000
- "syn 2.0.31",
3001
  ]
3002
 
 
 
 
 
 
 
3003
  [[package]]
3004
  name = "time"
3005
- version = "0.1.45"
3006
  source = "registry+https://github.com/rust-lang/crates.io-index"
3007
- checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a"
3008
  dependencies = [
3009
  "libc",
3010
- "wasi 0.10.0+wasi-snapshot-preview1",
3011
  "winapi 0.3.9",
3012
  ]
3013
 
@@ -3071,7 +3326,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3071
  checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
3072
  dependencies = [
3073
  "bytes 0.4.12",
3074
- "futures",
3075
  "mio 0.6.23",
3076
  "num_cpus",
3077
  "tokio-current-thread",
@@ -3090,7 +3345,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3090
  checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
3091
  dependencies = [
3092
  "backtrace",
3093
- "bytes 1.4.0",
3094
  "libc",
3095
  "mio 0.8.8",
3096
  "num_cpus",
@@ -3110,7 +3365,7 @@ checksum = "8fb220f46c53859a4b7ec083e41dec9778ff0b1851c0942b211edb89e0ccdc46"
3110
  dependencies = [
3111
  "bytes 0.4.12",
3112
  "either",
3113
- "futures",
3114
  ]
3115
 
3116
  [[package]]
@@ -3119,7 +3374,7 @@ version = "0.1.7"
3119
  source = "registry+https://github.com/rust-lang/crates.io-index"
3120
  checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e"
3121
  dependencies = [
3122
- "futures",
3123
  "tokio-executor",
3124
  ]
3125
 
@@ -3130,7 +3385,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3130
  checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671"
3131
  dependencies = [
3132
  "crossbeam-utils 0.7.2",
3133
- "futures",
3134
  ]
3135
 
3136
  [[package]]
@@ -3140,7 +3395,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3140
  checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
3141
  dependencies = [
3142
  "bytes 0.4.12",
3143
- "futures",
3144
  "log",
3145
  ]
3146
 
@@ -3152,7 +3407,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
3152
  dependencies = [
3153
  "proc-macro2 1.0.66",
3154
  "quote 1.0.33",
3155
- "syn 2.0.31",
3156
  ]
3157
 
3158
  [[package]]
@@ -3172,7 +3427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3172
  checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351"
3173
  dependencies = [
3174
  "crossbeam-utils 0.7.2",
3175
- "futures",
3176
  "lazy_static",
3177
  "log",
3178
  "mio 0.6.23",
@@ -3184,6 +3439,17 @@ dependencies = [
3184
  "tokio-sync",
3185
  ]
3186
 
 
 
 
 
 
 
 
 
 
 
 
3187
  [[package]]
3188
  name = "tokio-sync"
3189
  version = "0.1.8"
@@ -3191,7 +3457,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3191
  checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee"
3192
  dependencies = [
3193
  "fnv",
3194
- "futures",
3195
  ]
3196
 
3197
  [[package]]
@@ -3201,7 +3467,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3201
  checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72"
3202
  dependencies = [
3203
  "bytes 0.4.12",
3204
- "futures",
3205
  "iovec",
3206
  "mio 0.6.23",
3207
  "tokio-io",
@@ -3217,7 +3483,7 @@ dependencies = [
3217
  "crossbeam-deque 0.7.4",
3218
  "crossbeam-queue",
3219
  "crossbeam-utils 0.7.2",
3220
- "futures",
3221
  "lazy_static",
3222
  "log",
3223
  "num_cpus",
@@ -3232,7 +3498,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3232
  checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296"
3233
  dependencies = [
3234
  "crossbeam-utils 0.7.2",
3235
- "futures",
3236
  "slab",
3237
  "tokio-executor",
3238
  ]
@@ -3243,7 +3509,7 @@ version = "0.7.8"
3243
  source = "registry+https://github.com/rust-lang/crates.io-index"
3244
  checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
3245
  dependencies = [
3246
- "bytes 1.4.0",
3247
  "futures-core",
3248
  "futures-sink",
3249
  "pin-project-lite",
@@ -3413,9 +3679,9 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
3413
 
3414
  [[package]]
3415
  name = "walkdir"
3416
- version = "2.3.3"
3417
  source = "registry+https://github.com/rust-lang/crates.io-index"
3418
- checksum = "36df944cda56c7d8d8b7496af378e6b16de9284591917d307c9b4d313c44e698"
3419
  dependencies = [
3420
  "same-file",
3421
  "winapi-util",
@@ -3427,7 +3693,7 @@ version = "0.2.0"
3427
  source = "registry+https://github.com/rust-lang/crates.io-index"
3428
  checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
3429
  dependencies = [
3430
- "futures",
3431
  "log",
3432
  "try-lock",
3433
  ]
@@ -3443,9 +3709,9 @@ dependencies = [
3443
 
3444
  [[package]]
3445
  name = "wasi"
3446
- version = "0.10.0+wasi-snapshot-preview1"
3447
  source = "registry+https://github.com/rust-lang/crates.io-index"
3448
- checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"
3449
 
3450
  [[package]]
3451
  name = "wasi"
@@ -3474,7 +3740,7 @@ dependencies = [
3474
  "once_cell",
3475
  "proc-macro2 1.0.66",
3476
  "quote 1.0.33",
3477
- "syn 2.0.31",
3478
  "wasm-bindgen-shared",
3479
  ]
3480
 
@@ -3508,7 +3774,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b"
3508
  dependencies = [
3509
  "proc-macro2 1.0.66",
3510
  "quote 1.0.33",
3511
- "syn 2.0.31",
3512
  "wasm-bindgen-backend",
3513
  "wasm-bindgen-shared",
3514
  ]
@@ -3535,25 +3801,31 @@ version = "0.20.2"
3535
  dependencies = [
3536
  "actix-cors",
3537
  "actix-files",
 
3538
  "actix-web",
 
3539
  "async-trait",
3540
  "criterion",
 
3541
  "env_logger",
3542
  "error-stack",
3543
  "fake-useragent",
 
3544
  "handlebars",
3545
  "log",
3546
  "md5",
 
 
3547
  "once_cell",
3548
  "rand 0.8.5",
3549
  "redis",
3550
  "regex",
3551
  "reqwest 0.11.20",
3552
- "rlua",
3553
  "rusty-hook",
3554
  "scraper",
3555
  "serde",
3556
  "serde_json",
 
3557
  "tempfile",
3558
  "tokio 1.32.0",
3559
  ]
 
9
  checksum = "617a8268e3537fe1d8c9ead925fca49ef6400927ee7bc26750e90ecee14ce4b8"
10
  dependencies = [
11
  "bitflags 1.3.2",
12
+ "bytes 1.5.0",
13
  "futures-core",
14
  "futures-sink",
15
  "memchr",
 
46
  "actix-web",
47
  "askama_escape",
48
  "bitflags 1.3.2",
49
+ "bytes 1.5.0",
50
  "derive_more",
51
  "futures-core",
52
  "http-range",
 
57
  "pin-project-lite",
58
  ]
59
 
60
+ [[package]]
61
+ name = "actix-governor"
62
+ version = "0.4.1"
63
+ source = "registry+https://github.com/rust-lang/crates.io-index"
64
+ checksum = "46ff2d40f2bc627b8054c5e20fa6b0b0cf9428699b54bd41634e9ae3098ad555"
65
+ dependencies = [
66
+ "actix-http",
67
+ "actix-web",
68
+ "futures 0.3.28",
69
+ "governor",
70
+ ]
71
+
72
  [[package]]
73
  name = "actix-http"
74
  version = "3.4.0"
 
80
  "actix-service",
81
  "actix-utils",
82
  "ahash",
83
+ "base64 0.21.4",
84
  "bitflags 2.4.0",
85
  "brotli",
86
+ "bytes 1.5.0",
87
  "bytestring",
88
  "derive_more",
89
  "encoding_rs",
 
115
  checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb"
116
  dependencies = [
117
  "quote 1.0.33",
118
+ "syn 2.0.32",
119
  ]
120
 
121
  [[package]]
 
195
  "actix-utils",
196
  "actix-web-codegen",
197
  "ahash",
198
+ "bytes 1.5.0",
199
  "bytestring",
200
  "cfg-if 1.0.0",
201
  "cookie 0.16.2",
 
228
  "actix-router",
229
  "proc-macro2 1.0.66",
230
  "quote 1.0.33",
231
+ "syn 2.0.32",
232
  ]
233
 
234
  [[package]]
 
290
 
291
  [[package]]
292
  name = "anstyle"
293
+ version = "1.0.3"
294
  source = "registry+https://github.com/rust-lang/crates.io-index"
295
+ checksum = "b84bf0a05bbb2a83e5eb6fa36bb6e87baa08193c35ff52bbf6b38d8af2890e46"
296
 
297
  [[package]]
298
  name = "anyhow"
 
300
  source = "registry+https://github.com/rust-lang/crates.io-index"
301
  checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
302
 
303
+ [[package]]
304
+ name = "arc-swap"
305
+ version = "1.6.0"
306
+ source = "registry+https://github.com/rust-lang/crates.io-index"
307
+ checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6"
308
+
309
  [[package]]
310
  name = "askama_escape"
311
  version = "0.10.3"
312
  source = "registry+https://github.com/rust-lang/crates.io-index"
313
  checksum = "619743e34b5ba4e9703bba34deac3427c72507c7159f5fd030aea8cac0cfe341"
314
 
315
+ [[package]]
316
+ name = "async-once-cell"
317
+ version = "0.5.3"
318
+ source = "registry+https://github.com/rust-lang/crates.io-index"
319
+ checksum = "9338790e78aa95a416786ec8389546c4b6a1dfc3dc36071ed9518a9413a542eb"
320
+
321
  [[package]]
322
  name = "async-trait"
323
  version = "0.1.73"
 
326
  dependencies = [
327
  "proc-macro2 1.0.66",
328
  "quote 1.0.33",
329
+ "syn 2.0.32",
330
  ]
331
 
332
  [[package]]
 
370
 
371
  [[package]]
372
  name = "base64"
373
+ version = "0.21.4"
374
  source = "registry+https://github.com/rust-lang/crates.io-index"
375
+ checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2"
376
 
377
  [[package]]
378
  name = "bit-set"
 
465
 
466
  [[package]]
467
  name = "bytes"
468
+ version = "1.5.0"
469
  source = "registry+https://github.com/rust-lang/crates.io-index"
470
+ checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
471
 
472
  [[package]]
473
  name = "bytestring"
 
475
  source = "registry+https://github.com/rust-lang/crates.io-index"
476
  checksum = "238e4886760d98c4f899360c834fa93e62cf7f721ac3c2da375cbdf4b8679aae"
477
  dependencies = [
478
+ "bytes 1.5.0",
479
  ]
480
 
481
  [[package]]
 
582
  source = "registry+https://github.com/rust-lang/crates.io-index"
583
  checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
584
  dependencies = [
585
+ "bytes 1.5.0",
586
+ "futures-core",
587
  "memchr",
588
+ "pin-project-lite",
589
+ "tokio 1.32.0",
590
+ "tokio-util",
591
  ]
592
 
593
  [[package]]
 
602
  source = "registry+https://github.com/rust-lang/crates.io-index"
603
  checksum = "888604f00b3db336d2af898ec3c1d5d0ddf5e6d462220f2ededc33a87ac4bbd5"
604
  dependencies = [
605
+ "time 0.1.43",
606
  "url 1.7.2",
607
  ]
608
 
 
630
  "publicsuffix",
631
  "serde",
632
  "serde_json",
633
+ "time 0.1.43",
634
  "try_from",
635
  "url 1.7.2",
636
  ]
 
826
  checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
827
  dependencies = [
828
  "quote 1.0.33",
829
+ "syn 2.0.32",
830
+ ]
831
+
832
+ [[package]]
833
+ name = "dashmap"
834
+ version = "5.5.3"
835
+ source = "registry+https://github.com/rust-lang/crates.io-index"
836
+ checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856"
837
+ dependencies = [
838
+ "cfg-if 1.0.0",
839
+ "hashbrown 0.14.0",
840
+ "lock_api 0.4.10",
841
+ "once_cell",
842
+ "parking_lot_core 0.9.8",
843
  ]
844
 
845
  [[package]]
 
861
  "syn 1.0.109",
862
  ]
863
 
864
+ [[package]]
865
+ name = "dhat"
866
+ version = "0.3.2"
867
+ source = "registry+https://github.com/rust-lang/crates.io-index"
868
+ checksum = "4f2aaf837aaf456f6706cb46386ba8dffd4013a757e36f4ea05c20dd46b209a3"
869
+ dependencies = [
870
+ "backtrace",
871
+ "lazy_static",
872
+ "mintex",
873
+ "parking_lot 0.12.1",
874
+ "rustc-hash",
875
+ "serde",
876
+ "serde_json",
877
+ "thousands",
878
+ ]
879
+
880
  [[package]]
881
  name = "digest"
882
  version = "0.10.7"
 
975
 
976
  [[package]]
977
  name = "error-stack"
978
+ version = "0.4.1"
979
  source = "registry+https://github.com/rust-lang/crates.io-index"
980
+ checksum = "27a72baa257b5e0e2de241967bc5ee8f855d6072351042688621081d66b2a76b"
981
  dependencies = [
982
  "anyhow",
983
  "rustc_version 0.4.0",
 
1106
  source = "registry+https://github.com/rust-lang/crates.io-index"
1107
  checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
1108
 
1109
+ [[package]]
1110
+ name = "futures"
1111
+ version = "0.3.28"
1112
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1113
+ checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40"
1114
+ dependencies = [
1115
+ "futures-channel",
1116
+ "futures-core",
1117
+ "futures-executor",
1118
+ "futures-io",
1119
+ "futures-sink",
1120
+ "futures-task",
1121
+ "futures-util",
1122
+ ]
1123
+
1124
  [[package]]
1125
  name = "futures-channel"
1126
  version = "0.3.28"
 
1128
  checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2"
1129
  dependencies = [
1130
  "futures-core",
1131
+ "futures-sink",
1132
  ]
1133
 
1134
  [[package]]
 
1143
  source = "registry+https://github.com/rust-lang/crates.io-index"
1144
  checksum = "ab90cde24b3319636588d0c35fe03b1333857621051837ed769faefb4c2162e4"
1145
  dependencies = [
1146
+ "futures 0.1.31",
1147
  "num_cpus",
1148
  ]
1149
 
1150
+ [[package]]
1151
+ name = "futures-executor"
1152
+ version = "0.3.28"
1153
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1154
+ checksum = "ccecee823288125bd88b4d7f565c9e58e41858e47ab72e8ea2d64e93624386e0"
1155
+ dependencies = [
1156
+ "futures-core",
1157
+ "futures-task",
1158
+ "futures-util",
1159
+ ]
1160
+
1161
+ [[package]]
1162
+ name = "futures-io"
1163
+ version = "0.3.28"
1164
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1165
+ checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964"
1166
+
1167
+ [[package]]
1168
+ name = "futures-macro"
1169
+ version = "0.3.28"
1170
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1171
+ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72"
1172
+ dependencies = [
1173
+ "proc-macro2 1.0.66",
1174
+ "quote 1.0.33",
1175
+ "syn 2.0.32",
1176
+ ]
1177
+
1178
  [[package]]
1179
  name = "futures-sink"
1180
  version = "0.3.28"
 
1187
  source = "registry+https://github.com/rust-lang/crates.io-index"
1188
  checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65"
1189
 
1190
+ [[package]]
1191
+ name = "futures-timer"
1192
+ version = "3.0.2"
1193
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1194
+ checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c"
1195
+
1196
  [[package]]
1197
  name = "futures-util"
1198
  version = "0.3.28"
1199
  source = "registry+https://github.com/rust-lang/crates.io-index"
1200
  checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533"
1201
  dependencies = [
1202
+ "futures-channel",
1203
  "futures-core",
1204
+ "futures-io",
1205
+ "futures-macro",
1206
+ "futures-sink",
1207
  "futures-task",
1208
+ "memchr",
1209
  "pin-project-lite",
1210
  "pin-utils",
1211
+ "slab",
1212
  ]
1213
 
1214
  [[package]]
 
1256
  source = "registry+https://github.com/rust-lang/crates.io-index"
1257
  checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0"
1258
 
1259
+ [[package]]
1260
+ name = "governor"
1261
+ version = "0.5.1"
1262
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1263
+ checksum = "c390a940a5d157878dd057c78680a33ce3415bcd05b4799509ea44210914b4d5"
1264
+ dependencies = [
1265
+ "cfg-if 1.0.0",
1266
+ "dashmap",
1267
+ "futures 0.3.28",
1268
+ "futures-timer",
1269
+ "no-std-compat",
1270
+ "nonzero_ext",
1271
+ "parking_lot 0.12.1",
1272
+ "quanta",
1273
+ "rand 0.8.5",
1274
+ "smallvec 1.11.0",
1275
+ ]
1276
+
1277
  [[package]]
1278
  name = "h2"
1279
  version = "0.1.26"
 
1283
  "byteorder",
1284
  "bytes 0.4.12",
1285
  "fnv",
1286
+ "futures 0.1.31",
1287
  "http 0.1.21",
1288
  "indexmap",
1289
  "log",
 
1298
  source = "registry+https://github.com/rust-lang/crates.io-index"
1299
  checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833"
1300
  dependencies = [
1301
+ "bytes 1.5.0",
1302
  "fnv",
1303
  "futures-core",
1304
  "futures-sink",
 
1338
  source = "registry+https://github.com/rust-lang/crates.io-index"
1339
  checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
1340
 
1341
+ [[package]]
1342
+ name = "hashbrown"
1343
+ version = "0.14.0"
1344
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1345
+ checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a"
1346
+
1347
  [[package]]
1348
  name = "hermit-abi"
1349
  version = "0.3.2"
 
1395
  source = "registry+https://github.com/rust-lang/crates.io-index"
1396
  checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
1397
  dependencies = [
1398
+ "bytes 1.5.0",
1399
  "fnv",
1400
  "itoa 1.0.9",
1401
  ]
 
1407
  checksum = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d"
1408
  dependencies = [
1409
  "bytes 0.4.12",
1410
+ "futures 0.1.31",
1411
  "http 0.1.21",
1412
  "tokio-buf",
1413
  ]
 
1418
  source = "registry+https://github.com/rust-lang/crates.io-index"
1419
  checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
1420
  dependencies = [
1421
+ "bytes 1.5.0",
1422
  "http 0.2.9",
1423
  "pin-project-lite",
1424
  ]
 
1454
  checksum = "5c843caf6296fc1f93444735205af9ed4e109a539005abb2564ae1d6fad34c52"
1455
  dependencies = [
1456
  "bytes 0.4.12",
1457
+ "futures 0.1.31",
1458
  "futures-cpupool",
1459
  "h2 0.1.26",
1460
  "http 0.1.21",
 
1465
  "log",
1466
  "net2",
1467
  "rustc_version 0.2.3",
1468
+ "time 0.1.43",
1469
  "tokio 0.1.22",
1470
  "tokio-buf",
1471
  "tokio-executor",
 
1483
  source = "registry+https://github.com/rust-lang/crates.io-index"
1484
  checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468"
1485
  dependencies = [
1486
+ "bytes 1.5.0",
1487
  "futures-channel",
1488
  "futures-core",
1489
  "futures-util",
 
1508
  checksum = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f"
1509
  dependencies = [
1510
  "bytes 0.4.12",
1511
+ "futures 0.1.31",
1512
  "hyper 0.12.36",
1513
  "native-tls",
1514
  "tokio-io",
 
1520
  source = "registry+https://github.com/rust-lang/crates.io-index"
1521
  checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
1522
  dependencies = [
1523
+ "bytes 1.5.0",
1524
  "hyper 0.14.27",
1525
  "native-tls",
1526
  "tokio 1.32.0",
 
1566
  checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
1567
  dependencies = [
1568
  "autocfg 1.1.0",
1569
+ "hashbrown 0.12.3",
1570
  ]
1571
 
1572
  [[package]]
 
1662
  source = "registry+https://github.com/rust-lang/crates.io-index"
1663
  checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3"
1664
 
1665
+ [[package]]
1666
+ name = "libmimalloc-sys"
1667
+ version = "0.1.34"
1668
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1669
+ checksum = "25d058a81af0d1c22d7a1c948576bee6d673f7af3c0f35564abd6c81122f513d"
1670
+ dependencies = [
1671
+ "cc",
1672
+ "libc",
1673
+ ]
1674
+
1675
  [[package]]
1676
  name = "linux-raw-sys"
1677
+ version = "0.4.7"
1678
  source = "registry+https://github.com/rust-lang/crates.io-index"
1679
+ checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128"
1680
 
1681
  [[package]]
1682
  name = "local-channel"
 
1727
  source = "registry+https://github.com/rust-lang/crates.io-index"
1728
  checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
1729
 
1730
+ [[package]]
1731
+ name = "mach"
1732
+ version = "0.3.2"
1733
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1734
+ checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa"
1735
+ dependencies = [
1736
+ "libc",
1737
+ ]
1738
+
1739
  [[package]]
1740
  name = "markup5ever"
1741
  version = "0.8.1"
 
1809
  "autocfg 1.1.0",
1810
  ]
1811
 
1812
+ [[package]]
1813
+ name = "mimalloc"
1814
+ version = "0.1.38"
1815
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1816
+ checksum = "972e5f23f6716f62665760b0f4cbf592576a80c7b879ba9beaafc0e558894127"
1817
+ dependencies = [
1818
+ "libmimalloc-sys",
1819
+ ]
1820
+
1821
  [[package]]
1822
  name = "mime"
1823
  version = "0.3.17"
 
1843
  "adler",
1844
  ]
1845
 
1846
+ [[package]]
1847
+ name = "mintex"
1848
+ version = "0.1.2"
1849
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1850
+ checksum = "fd7c5ba1c3b5a23418d7bbf98c71c3d4946a0125002129231da8d6b723d559cb"
1851
+ dependencies = [
1852
+ "once_cell",
1853
+ "sys-info",
1854
+ ]
1855
+
1856
  [[package]]
1857
  name = "mio"
1858
  version = "0.6.23"
 
1896
  "ws2_32-sys",
1897
  ]
1898
 
1899
+ [[package]]
1900
+ name = "mlua"
1901
+ version = "0.8.10"
1902
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1903
+ checksum = "0bb37b0ba91f017aa7ca2b98ef99496827770cd635b4a932a6047c5b4bbe678e"
1904
+ dependencies = [
1905
+ "bstr",
1906
+ "cc",
1907
+ "num-traits",
1908
+ "once_cell",
1909
+ "pkg-config",
1910
+ "rustc-hash",
1911
+ ]
1912
+
1913
  [[package]]
1914
  name = "native-tls"
1915
  version = "0.2.11"
 
1951
  source = "registry+https://github.com/rust-lang/crates.io-index"
1952
  checksum = "ab250442c86f1850815b5d268639dff018c0627022bc1940eb2d642ca1ce12f0"
1953
 
1954
+ [[package]]
1955
+ name = "no-std-compat"
1956
+ version = "0.4.1"
1957
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1958
+ checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c"
1959
+
1960
+ [[package]]
1961
+ name = "nonzero_ext"
1962
+ version = "0.3.0"
1963
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1964
+ checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21"
1965
+
1966
  [[package]]
1967
  name = "num-traits"
1968
  version = "0.2.16"
 
2026
  dependencies = [
2027
  "proc-macro2 1.0.66",
2028
  "quote 1.0.33",
2029
+ "syn 2.0.32",
2030
  ]
2031
 
2032
  [[package]]
 
2037
 
2038
  [[package]]
2039
  name = "openssl-sys"
2040
+ version = "0.9.93"
2041
  source = "registry+https://github.com/rust-lang/crates.io-index"
2042
+ checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d"
2043
  dependencies = [
2044
  "cc",
2045
  "libc",
 
2145
  "pest_meta",
2146
  "proc-macro2 1.0.66",
2147
  "quote 1.0.33",
2148
+ "syn 2.0.32",
2149
  ]
2150
 
2151
  [[package]]
 
2247
  "phf_shared 0.11.2",
2248
  "proc-macro2 1.0.66",
2249
  "quote 1.0.33",
2250
+ "syn 2.0.32",
2251
  ]
2252
 
2253
  [[package]]
 
2277
  "siphasher 0.3.11",
2278
  ]
2279
 
2280
+ [[package]]
2281
+ name = "pin-project"
2282
+ version = "1.1.3"
2283
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2284
+ checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
2285
+ dependencies = [
2286
+ "pin-project-internal",
2287
+ ]
2288
+
2289
+ [[package]]
2290
+ name = "pin-project-internal"
2291
+ version = "1.1.3"
2292
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2293
+ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
2294
+ dependencies = [
2295
+ "proc-macro2 1.0.66",
2296
+ "quote 1.0.33",
2297
+ "syn 2.0.32",
2298
+ ]
2299
+
2300
  [[package]]
2301
  name = "pin-project-lite"
2302
  version = "0.2.13"
 
2383
  "url 2.4.1",
2384
  ]
2385
 
2386
+ [[package]]
2387
+ name = "quanta"
2388
+ version = "0.9.3"
2389
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2390
+ checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8"
2391
+ dependencies = [
2392
+ "crossbeam-utils 0.8.16",
2393
+ "libc",
2394
+ "mach",
2395
+ "once_cell",
2396
+ "raw-cpuid",
2397
+ "wasi 0.10.2+wasi-snapshot-preview1",
2398
+ "web-sys",
2399
+ "winapi 0.3.9",
2400
+ ]
2401
+
2402
  [[package]]
2403
  name = "quote"
2404
  version = "0.6.13"
 
2553
  "rand_core 0.3.1",
2554
  ]
2555
 
2556
+ [[package]]
2557
+ name = "raw-cpuid"
2558
+ version = "10.7.0"
2559
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2560
+ checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
2561
+ dependencies = [
2562
+ "bitflags 1.3.2",
2563
+ ]
2564
+
2565
  [[package]]
2566
  name = "rayon"
2567
  version = "1.7.0"
 
2599
  source = "registry+https://github.com/rust-lang/crates.io-index"
2600
  checksum = "4f49cdc0bb3f412bf8e7d1bd90fe1d9eb10bc5c399ba90973c14662a27b3f8ba"
2601
  dependencies = [
2602
+ "arc-swap",
2603
+ "async-trait",
2604
+ "bytes 1.5.0",
2605
  "combine",
2606
+ "futures 0.3.28",
2607
+ "futures-util",
2608
  "itoa 1.0.9",
2609
  "percent-encoding 2.3.0",
2610
+ "pin-project-lite",
2611
  "ryu",
2612
  "sha1_smol",
2613
  "socket2 0.4.9",
2614
+ "tokio 1.32.0",
2615
+ "tokio-retry",
2616
+ "tokio-util",
2617
  "url 2.4.1",
2618
  ]
2619
 
 
2673
  "cookie_store",
2674
  "encoding_rs",
2675
  "flate2",
2676
+ "futures 0.1.31",
2677
  "http 0.1.21",
2678
  "hyper 0.12.36",
2679
  "hyper-tls 0.3.2",
 
2684
  "serde",
2685
  "serde_json",
2686
  "serde_urlencoded 0.5.5",
2687
+ "time 0.1.43",
2688
  "tokio 0.1.22",
2689
  "tokio-executor",
2690
  "tokio-io",
 
2701
  source = "registry+https://github.com/rust-lang/crates.io-index"
2702
  checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1"
2703
  dependencies = [
2704
+ "base64 0.21.4",
2705
+ "bytes 1.5.0",
2706
  "encoding_rs",
2707
  "futures-core",
2708
  "futures-util",
 
2732
  "winreg 0.50.0",
2733
  ]
2734
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2735
  [[package]]
2736
  name = "rustc-demangle"
2737
  version = "0.1.23"
2738
  source = "registry+https://github.com/rust-lang/crates.io-index"
2739
  checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
2740
 
2741
+ [[package]]
2742
+ name = "rustc-hash"
2743
+ version = "1.1.0"
2744
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2745
+ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
2746
+
2747
  [[package]]
2748
  name = "rustc_version"
2749
  version = "0.2.3"
 
2764
 
2765
  [[package]]
2766
  name = "rustix"
2767
+ version = "0.38.13"
2768
  source = "registry+https://github.com/rust-lang/crates.io-index"
2769
+ checksum = "d7db8590df6dfcd144d22afd1b83b36c21a18d7cbc1dc4bb5295a8712e9eb662"
2770
  dependencies = [
2771
  "bitflags 2.4.0",
2772
  "errno",
 
2924
  dependencies = [
2925
  "proc-macro2 1.0.66",
2926
  "quote 1.0.33",
2927
+ "syn 2.0.32",
2928
  ]
2929
 
2930
  [[package]]
2931
  name = "serde_json"
2932
+ version = "1.0.106"
2933
  source = "registry+https://github.com/rust-lang/crates.io-index"
2934
+ checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2"
2935
  dependencies = [
2936
  "itoa 1.0.9",
2937
  "ryu",
 
3043
  version = "1.11.0"
3044
  source = "registry+https://github.com/rust-lang/crates.io-index"
3045
  checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9"
3046
+ dependencies = [
3047
+ "serde",
3048
+ ]
3049
 
3050
  [[package]]
3051
  name = "socket2"
 
3166
 
3167
  [[package]]
3168
  name = "syn"
3169
+ version = "2.0.32"
3170
  source = "registry+https://github.com/rust-lang/crates.io-index"
3171
+ checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2"
3172
  dependencies = [
3173
  "proc-macro2 1.0.66",
3174
  "quote 1.0.33",
 
3187
  "unicode-xid 0.2.4",
3188
  ]
3189
 
3190
+ [[package]]
3191
+ name = "sys-info"
3192
+ version = "0.9.1"
3193
+ source = "registry+https://github.com/rust-lang/crates.io-index"
3194
+ checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c"
3195
+ dependencies = [
3196
+ "cc",
3197
+ "libc",
3198
+ ]
3199
+
3200
  [[package]]
3201
  name = "tempfile"
3202
  version = "3.8.0"
 
3247
  dependencies = [
3248
  "proc-macro2 1.0.66",
3249
  "quote 1.0.33",
3250
+ "syn 2.0.32",
3251
  ]
3252
 
3253
+ [[package]]
3254
+ name = "thousands"
3255
+ version = "0.2.0"
3256
+ source = "registry+https://github.com/rust-lang/crates.io-index"
3257
+ checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820"
3258
+
3259
  [[package]]
3260
  name = "time"
3261
+ version = "0.1.43"
3262
  source = "registry+https://github.com/rust-lang/crates.io-index"
3263
+ checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
3264
  dependencies = [
3265
  "libc",
 
3266
  "winapi 0.3.9",
3267
  ]
3268
 
 
3326
  checksum = "5a09c0b5bb588872ab2f09afa13ee6e9dac11e10a0ec9e8e3ba39a5a5d530af6"
3327
  dependencies = [
3328
  "bytes 0.4.12",
3329
+ "futures 0.1.31",
3330
  "mio 0.6.23",
3331
  "num_cpus",
3332
  "tokio-current-thread",
 
3345
  checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9"
3346
  dependencies = [
3347
  "backtrace",
3348
+ "bytes 1.5.0",
3349
  "libc",
3350
  "mio 0.8.8",
3351
  "num_cpus",
 
3365
  dependencies = [
3366
  "bytes 0.4.12",
3367
  "either",
3368
+ "futures 0.1.31",
3369
  ]
3370
 
3371
  [[package]]
 
3374
  source = "registry+https://github.com/rust-lang/crates.io-index"
3375
  checksum = "b1de0e32a83f131e002238d7ccde18211c0a5397f60cbfffcb112868c2e0e20e"
3376
  dependencies = [
3377
+ "futures 0.1.31",
3378
  "tokio-executor",
3379
  ]
3380
 
 
3385
  checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671"
3386
  dependencies = [
3387
  "crossbeam-utils 0.7.2",
3388
+ "futures 0.1.31",
3389
  ]
3390
 
3391
  [[package]]
 
3395
  checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
3396
  dependencies = [
3397
  "bytes 0.4.12",
3398
+ "futures 0.1.31",
3399
  "log",
3400
  ]
3401
 
 
3407
  dependencies = [
3408
  "proc-macro2 1.0.66",
3409
  "quote 1.0.33",
3410
+ "syn 2.0.32",
3411
  ]
3412
 
3413
  [[package]]
 
3427
  checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351"
3428
  dependencies = [
3429
  "crossbeam-utils 0.7.2",
3430
+ "futures 0.1.31",
3431
  "lazy_static",
3432
  "log",
3433
  "mio 0.6.23",
 
3439
  "tokio-sync",
3440
  ]
3441
 
3442
+ [[package]]
3443
+ name = "tokio-retry"
3444
+ version = "0.3.0"
3445
+ source = "registry+https://github.com/rust-lang/crates.io-index"
3446
+ checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
3447
+ dependencies = [
3448
+ "pin-project",
3449
+ "rand 0.8.5",
3450
+ "tokio 1.32.0",
3451
+ ]
3452
+
3453
  [[package]]
3454
  name = "tokio-sync"
3455
  version = "0.1.8"
 
3457
  checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee"
3458
  dependencies = [
3459
  "fnv",
3460
+ "futures 0.1.31",
3461
  ]
3462
 
3463
  [[package]]
 
3467
  checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72"
3468
  dependencies = [
3469
  "bytes 0.4.12",
3470
+ "futures 0.1.31",
3471
  "iovec",
3472
  "mio 0.6.23",
3473
  "tokio-io",
 
3483
  "crossbeam-deque 0.7.4",
3484
  "crossbeam-queue",
3485
  "crossbeam-utils 0.7.2",
3486
+ "futures 0.1.31",
3487
  "lazy_static",
3488
  "log",
3489
  "num_cpus",
 
3498
  checksum = "93044f2d313c95ff1cb7809ce9a7a05735b012288a888b62d4434fd58c94f296"
3499
  dependencies = [
3500
  "crossbeam-utils 0.7.2",
3501
+ "futures 0.1.31",
3502
  "slab",
3503
  "tokio-executor",
3504
  ]
 
3509
  source = "registry+https://github.com/rust-lang/crates.io-index"
3510
  checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
3511
  dependencies = [
3512
+ "bytes 1.5.0",
3513
  "futures-core",
3514
  "futures-sink",
3515
  "pin-project-lite",
 
3679
 
3680
  [[package]]
3681
  name = "walkdir"
3682
+ version = "2.4.0"
3683
  source = "registry+https://github.com/rust-lang/crates.io-index"
3684
+ checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
3685
  dependencies = [
3686
  "same-file",
3687
  "winapi-util",
 
3693
  source = "registry+https://github.com/rust-lang/crates.io-index"
3694
  checksum = "b6395efa4784b027708f7451087e647ec73cc74f5d9bc2e418404248d679a230"
3695
  dependencies = [
3696
+ "futures 0.1.31",
3697
  "log",
3698
  "try-lock",
3699
  ]
 
3709
 
3710
  [[package]]
3711
  name = "wasi"
3712
+ version = "0.10.2+wasi-snapshot-preview1"
3713
  source = "registry+https://github.com/rust-lang/crates.io-index"
3714
+ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
3715
 
3716
  [[package]]
3717
  name = "wasi"
 
3740
  "once_cell",
3741
  "proc-macro2 1.0.66",
3742
  "quote 1.0.33",
3743
+ "syn 2.0.32",
3744
  "wasm-bindgen-shared",
3745
  ]
3746
 
 
3774
  dependencies = [
3775
  "proc-macro2 1.0.66",
3776
  "quote 1.0.33",
3777
+ "syn 2.0.32",
3778
  "wasm-bindgen-backend",
3779
  "wasm-bindgen-shared",
3780
  ]
 
3801
  dependencies = [
3802
  "actix-cors",
3803
  "actix-files",
3804
+ "actix-governor",
3805
  "actix-web",
3806
+ "async-once-cell",
3807
  "async-trait",
3808
  "criterion",
3809
+ "dhat",
3810
  "env_logger",
3811
  "error-stack",
3812
  "fake-useragent",
3813
+ "futures 0.3.28",
3814
  "handlebars",
3815
  "log",
3816
  "md5",
3817
+ "mimalloc",
3818
+ "mlua",
3819
  "once_cell",
3820
  "rand 0.8.5",
3821
  "redis",
3822
  "regex",
3823
  "reqwest 0.11.20",
 
3824
  "rusty-hook",
3825
  "scraper",
3826
  "serde",
3827
  "serde_json",
3828
+ "smallvec 1.11.0",
3829
  "tempfile",
3830
  "tokio 1.32.0",
3831
  ]
Cargo.toml CHANGED
@@ -8,9 +8,9 @@ license = "AGPL-3.0"
8
 
9
  [dependencies]
10
  reqwest = {version="0.11.20",features=["json"]}
11
- tokio = {version="1.32.0",features=["full"]}
12
  serde = {version="1.0.188",features=["derive"]}
13
- handlebars = { version = "4.3.7", features = ["dir_source"] }
14
  scraper = {version="0.17.1"}
15
  actix-web = {version="4.4.0", features = ["cookies"]}
16
  actix-files = {version="0.6.2"}
@@ -19,14 +19,20 @@ serde_json = {version="1.0.105"}
19
  fake-useragent = {version="0.1.3"}
20
  env_logger = {version="0.10.0"}
21
  log = {version="0.4.20"}
22
- rlua = {version="0.19.7"}
23
- redis = {version="0.23.2"}
24
  md5 = {version="0.7.0"}
25
  rand={version="0.8.5"}
26
  once_cell = {version="1.18.0"}
27
  error-stack = {version="0.4.0"}
28
  async-trait = {version="0.1.73"}
29
  regex = {version="1.9.4", features=["perf"]}
 
 
 
 
 
 
30
 
31
  [dev-dependencies]
32
  rusty-hook = "^0.11.2"
@@ -47,13 +53,17 @@ rpath = false
47
 
48
  [profile.release]
49
  opt-level = 3
50
- debug = false
 
51
  split-debuginfo = '...'
52
  debug-assertions = false
53
  overflow-checks = false
54
- lto = 'thin'
55
  panic = 'abort'
56
  incremental = false
57
- codegen-units = 16
58
  rpath = false
59
  strip = "debuginfo"
 
 
 
 
8
 
9
  [dependencies]
10
  reqwest = {version="0.11.20",features=["json"]}
11
+ tokio = {version="1.32.0",features=["rt-multi-thread","macros"]}
12
  serde = {version="1.0.188",features=["derive"]}
13
+ handlebars = { version = "4.4.0", features = ["dir_source"] }
14
  scraper = {version="0.17.1"}
15
  actix-web = {version="4.4.0", features = ["cookies"]}
16
  actix-files = {version="0.6.2"}
 
19
  fake-useragent = {version="0.1.3"}
20
  env_logger = {version="0.10.0"}
21
  log = {version="0.4.20"}
22
+ mlua = {version="0.8.10", features=["luajit"]}
23
+ redis = {version="0.23.3", features=["tokio-comp","connection-manager"]}
24
  md5 = {version="0.7.0"}
25
  rand={version="0.8.5"}
26
  once_cell = {version="1.18.0"}
27
  error-stack = {version="0.4.0"}
28
  async-trait = {version="0.1.73"}
29
  regex = {version="1.9.4", features=["perf"]}
30
+ smallvec = {version="1.11.0", features=["union", "serde"]}
31
+ futures = {version="0.3.28"}
32
+ dhat = {version="0.3.2", optional = true}
33
+ mimalloc = { version = "0.1.38", default-features = false }
34
+ async-once-cell = {version="0.5.3"}
35
+ actix-governor = {version="0.4.1"}
36
 
37
  [dev-dependencies]
38
  rusty-hook = "^0.11.2"
 
53
 
54
  [profile.release]
55
  opt-level = 3
56
+ debug = false # This should only be commented when testing with dhat profiler
57
+ # debug = 1 # This should only be uncommented when testing with dhat profiler
58
  split-debuginfo = '...'
59
  debug-assertions = false
60
  overflow-checks = false
61
+ lto = true
62
  panic = 'abort'
63
  incremental = false
64
+ codegen-units = 1
65
  rpath = false
66
  strip = "debuginfo"
67
+
68
+ [features]
69
+ dhat-heap = ["dep:dhat"]
Dockerfile CHANGED
@@ -19,7 +19,7 @@ COPY . .
19
  RUN cargo install --path .
20
 
21
  # We do not need the Rust toolchain to run the binary!
22
- FROM gcr.io/distroless/cc-debian11
23
  COPY --from=builder /app/public/ /opt/websurfx/public/
24
  COPY --from=builder /app/websurfx/config.lua /etc/xdg/websurfx/config.lua
25
  COPY --from=builder /usr/local/cargo/bin/* /usr/local/bin/
 
19
  RUN cargo install --path .
20
 
21
  # We do not need the Rust toolchain to run the binary!
22
+ FROM gcr.io/distroless/cc-debian12
23
  COPY --from=builder /app/public/ /opt/websurfx/public/
24
  COPY --from=builder /app/websurfx/config.lua /etc/xdg/websurfx/config.lua
25
  COPY --from=builder /usr/local/cargo/bin/* /usr/local/bin/
README.md CHANGED
@@ -5,7 +5,7 @@
5
  <b align="center"><a href="README.md">Readme</a></b> |
6
  <b><a href="https://discord.gg/SWnda7Mw5u">Discord</a></b> |
7
  <b><a href="https://github.com/neon-mmd/websurfx">GitHub</a></b> |
8
- <b><a href="./docs/README.md">Documentation</a></b>
9
  <br /><br />
10
  <a href="#">
11
  <img
@@ -51,7 +51,7 @@
51
  - **Getting Started**
52
  - [🔭 Preview](#preview-)
53
  - [🚀 Features](#features-)
54
- - [🛠️ Installation and Testing](#installation-and-testing-)
55
  - [🔧 Configuration](#configuration-)
56
  - **Feature Overview**
57
  - [🎨 Theming](#theming-)
 
5
  <b align="center"><a href="README.md">Readme</a></b> |
6
  <b><a href="https://discord.gg/SWnda7Mw5u">Discord</a></b> |
7
  <b><a href="https://github.com/neon-mmd/websurfx">GitHub</a></b> |
8
+ <b><a href="../../tree/HEAD/docs/">Documentation</a></b>
9
  <br /><br />
10
  <a href="#">
11
  <img
 
51
  - **Getting Started**
52
  - [🔭 Preview](#preview-)
53
  - [🚀 Features](#features-)
54
+ - [🛠️ Installation and Testing](#installation-and-testing-%EF%B8%8F)
55
  - [🔧 Configuration](#configuration-)
56
  - **Feature Overview**
57
  - [🎨 Theming](#theming-)
docs/installation.md CHANGED
@@ -109,7 +109,7 @@ colorscheme = "catppuccin-mocha" -- the colorscheme name which should be used fo
109
  theme = "simple" -- the theme name which should be used for the website
110
 
111
  -- ### Caching ###
112
- redis_connection_url = "redis://redis:6379" -- redis connection url address on which the client should connect on.
113
 
114
  -- ### Search Engines ###
115
  upstream_search_engines = { DuckDuckGo = true, Searx = false } -- select the upstream search engines from which the results should be fetched.
 
109
  theme = "simple" -- the theme name which should be used for the website
110
 
111
  -- ### Caching ###
112
+ redis_url = "redis://redis:6379" -- redis connection url address on which the client should connect on.
113
 
114
  -- ### Search Engines ###
115
  upstream_search_engines = { DuckDuckGo = true, Searx = false } -- select the upstream search engines from which the results should be fetched.
public/images/barricade.png ADDED
public/images/filter.png ADDED
public/static/themes/simple.css CHANGED
@@ -132,6 +132,35 @@ body {
132
  width: 1.2rem;
133
  height: 1.2rem;
134
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
135
 
136
  /* styles for the footer and header */
137
 
 
132
  width: 1.2rem;
133
  height: 1.2rem;
134
  }
135
+ .results .result_disallowed,
136
+ .results .result_filtered {
137
+ display: flex;
138
+ justify-content: center;
139
+ align-items: center;
140
+ gap: 10rem;
141
+ font-size: 2rem;
142
+ color: var(--foreground-color);
143
+ margin: 0rem 7rem;
144
+ }
145
+
146
+ .results .result_disallowed .user_query,
147
+ .results .result_filtered .user_query {
148
+ color: var(--background-color);
149
+ font-weight: 300;
150
+ }
151
+
152
+ .results .result_disallowed img,
153
+ .results .result_filtered img {
154
+ width: 30rem;
155
+ }
156
+
157
+ .results .result_disallowed div,
158
+ .results .result_filtered div {
159
+ display: flex;
160
+ flex-direction: column;
161
+ gap: 1rem;
162
+ line-break: strict;
163
+ }
164
 
165
  /* styles for the footer and header */
166
 
public/templates/search.html CHANGED
@@ -1,37 +1,69 @@
1
  {{>header this.style}}
2
  <main class="results">
3
- {{>search_bar this}}
4
- <div class="results_aggregated">
5
- {{#if results}} {{#each results}}
6
- <div class="result">
7
- <h1><a href="{{{this.url}}}">{{{this.title}}}</a></h1>
8
- <small>{{{this.url}}}</small>
9
- <p>{{{this.description}}}</p>
10
- <div class="upstream_engines">
11
- {{#each engine}}
12
- <span>{{{this}}}</span>
13
- {{/each}}
14
- </div>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
15
  </div>
16
- {{/each}} {{else}}
17
- <div class="result_not_found">
18
- <p>Your search - {{{this.pageQuery}}} - did not match any documents.</p>
19
- <p class="suggestions">Suggestions:</p>
20
- <ul>
21
- <li>Make sure that all words are spelled correctly.</li>
22
- <li>Try different keywords.</li>
23
- <li>Try more general keywords.</li>
24
- </ul>
25
- <img src="./images/no_results.gif" alt="Man fishing gif" />
26
  </div>
27
- {{/if}}
28
- </div>
29
- <div class="page_navigation">
30
- <button type="button" onclick="navigate_backward()">
31
- &#8592; previous
32
- </button>
33
- <button type="button" onclick="navigate_forward()">next &#8594;</button>
34
- </div>
35
  </main>
36
  <script src="static/index.js"></script>
37
  <script src="static/pagination.js"></script>
 
1
  {{>header this.style}}
2
  <main class="results">
3
+ {{>search_bar this}}
4
+ <div class="results_aggregated">
5
+ {{#if results}} {{#each results}}
6
+ <div class="result">
7
+ <h1><a href="{{{this.url}}}">{{{this.title}}}</a></h1>
8
+ <small>{{{this.url}}}</small>
9
+ <p>{{{this.description}}}</p>
10
+ <div class="upstream_engines">
11
+ {{#each engine}}
12
+ <span>{{{this}}}</span>
13
+ {{/each}}
14
+ </div>
15
+ </div>
16
+ {{/each}} {{else}} {{#if disallowed}}
17
+ <div class="result_disallowed">
18
+ <div class="description">
19
+ <p>
20
+ Your search - <span class="user_query">{{{this.pageQuery}}}</span> -
21
+ has been disallowed.
22
+ </p>
23
+ <p class="description_paragraph">Dear user,</p>
24
+ <p class="description_paragraph">
25
+ The query - <span class="user_query">{{{this.pageQuery}}}</span> - has
26
+ been blacklisted via server configuration and hence disallowed by the
27
+ server. Henceforth no results could be displayed for your query.
28
+ </p>
29
+ </div>
30
+ <img src="./images/barricade.png" alt="Image of a Barricade" />
31
+ </div>
32
+ {{else}} {{#if filtered}}
33
+ <div class="result_filtered">
34
+ <div class="description">
35
+ <p>
36
+ Your search - <span class="user_query">{{{this.pageQuery}}}</span> -
37
+ has been filtered.
38
+ </p>
39
+ <p class="description_paragraph">Dear user,</p>
40
+ <p class="description_paragraph">
41
+ All the search results contain results that has been configured to be
42
+ filtered out via server configuration and henceforth has been
43
+ completely filtered out.
44
+ </p>
45
+ </div>
46
+ <img src="./images/filter.png" alt="Image of a paper inside a funnel" />
47
+ </div>
48
+ {{else}}
49
+ <div class="result_not_found">
50
+ <p>Your search - {{{this.pageQuery}}} - did not match any documents.</p>
51
+ <p class="suggestions">Suggestions:</p>
52
+ <ul>
53
+ <li>Make sure that all words are spelled correctly.</li>
54
+ <li>Try different keywords.</li>
55
+ <li>Try more general keywords.</li>
56
+ </ul>
57
+ <img src="./images/no_results.gif" alt="Man fishing gif" />
58
+ </div>
59
+ {{/if}} {{/if}} {{/if}}
60
  </div>
61
+ <div class="page_navigation">
62
+ <button type="button" onclick="navigate_backward()">
63
+ &#8592; previous
64
+ </button>
65
+ <button type="button" onclick="navigate_forward()">next &#8594;</button>
 
 
 
 
 
66
  </div>
 
 
 
 
 
 
 
 
67
  </main>
68
  <script src="static/index.js"></script>
69
  <script src="static/pagination.js"></script>
src/bin/websurfx.rs CHANGED
@@ -3,9 +3,19 @@
3
  //! This module contains the main function which handles the logging of the application to the
4
  //! stdout and handles the command line arguments provided and launches the `websurfx` server.
5
 
 
6
  use std::net::TcpListener;
7
  use websurfx::{config::parser::Config, run};
8
 
 
 
 
 
 
 
 
 
 
9
  /// The function that launches the main server and registers all the routes of the website.
10
  ///
11
  /// # Error
@@ -14,6 +24,10 @@ use websurfx::{config::parser::Config, run};
14
  /// available for being used for other applications.
15
  #[actix_web::main]
16
  async fn main() -> std::io::Result<()> {
 
 
 
 
17
  // Initialize the parsed config file.
18
  let config = Config::parse(false).unwrap();
19
 
 
3
  //! This module contains the main function which handles the logging of the application to the
4
  //! stdout and handles the command line arguments provided and launches the `websurfx` server.
5
 
6
+ use mimalloc::MiMalloc;
7
  use std::net::TcpListener;
8
  use websurfx::{config::parser::Config, run};
9
 
10
+ /// A dhat heap memory profiler
11
+ #[cfg(feature = "dhat-heap")]
12
+ #[global_allocator]
13
+ static ALLOC: dhat::Alloc = dhat::Alloc;
14
+
15
+ #[cfg(not(feature = "dhat-heap"))]
16
+ #[global_allocator]
17
+ static GLOBAL: MiMalloc = MiMalloc;
18
+
19
  /// The function that launches the main server and registers all the routes of the website.
20
  ///
21
  /// # Error
 
24
  /// available for being used for other applications.
25
  #[actix_web::main]
26
  async fn main() -> std::io::Result<()> {
27
+ // A dhat heap profiler initialization.
28
+ #[cfg(feature = "dhat-heap")]
29
+ let _profiler = dhat::Profiler::new_heap();
30
+
31
  // Initialize the parsed config file.
32
  let config = Config::parse(false).unwrap();
33
 
src/cache/cacher.rs CHANGED
@@ -1,14 +1,24 @@
1
  //! This module provides the functionality to cache the aggregated results fetched and aggregated
2
  //! from the upstream search engines in a json format.
3
 
 
 
4
  use md5::compute;
5
- use redis::{Client, Commands, Connection};
 
 
6
 
7
  /// A named struct which stores the redis Connection url address to which the client will
8
  /// connect to.
 
9
  pub struct RedisCache {
10
- /// It stores the redis Connection url address.
11
- connection: Connection,
 
 
 
 
 
12
  }
13
 
14
  impl RedisCache {
@@ -16,11 +26,25 @@ impl RedisCache {
16
  ///
17
  /// # Arguments
18
  ///
19
- /// * `redis_connection_url` - It stores the redis Connection url address.
20
- pub fn new(redis_connection_url: String) -> Result<Self, Box<dyn std::error::Error>> {
 
 
 
 
 
21
  let client = Client::open(redis_connection_url)?;
22
- let connection = client.get_connection()?;
23
- let redis_cache = RedisCache { connection };
 
 
 
 
 
 
 
 
 
24
  Ok(redis_cache)
25
  }
26
 
@@ -29,7 +53,7 @@ impl RedisCache {
29
  /// # Arguments
30
  ///
31
  /// * `url` - It takes an url as string.
32
- fn hash_url(url: &str) -> String {
33
  format!("{:?}", compute(url))
34
  }
35
 
@@ -38,9 +62,42 @@ impl RedisCache {
38
  /// # Arguments
39
  ///
40
  /// * `url` - It takes an url as a string.
41
- pub fn cached_json(&mut self, url: &str) -> Result<String, Box<dyn std::error::Error>> {
42
- let hashed_url_string = Self::hash_url(url);
43
- Ok(self.connection.get(hashed_url_string)?)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
44
  }
45
 
46
  /// A function which caches the results by using the hashed `url` as the key and
@@ -51,21 +108,45 @@ impl RedisCache {
51
  ///
52
  /// * `json_results` - It takes the json results string as an argument.
53
  /// * `url` - It takes the url as a String.
54
- pub fn cache_results(
55
  &mut self,
56
- json_results: String,
57
  url: &str,
58
- ) -> Result<(), Box<dyn std::error::Error>> {
59
- let hashed_url_string = Self::hash_url(url);
60
-
61
- // put results_json into cache
62
- self.connection.set(&hashed_url_string, json_results)?;
63
 
64
- // Set the TTL for the key to 60 seconds
65
- self.connection
66
- .expire::<String, u32>(hashed_url_string, 60)
67
- .unwrap();
68
 
69
- Ok(())
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
  }
71
  }
 
1
  //! This module provides the functionality to cache the aggregated results fetched and aggregated
2
  //! from the upstream search engines in a json format.
3
 
4
+ use error_stack::Report;
5
+ use futures::future::try_join_all;
6
  use md5::compute;
7
+ use redis::{aio::ConnectionManager, AsyncCommands, Client, RedisError};
8
+
9
+ use super::error::PoolError;
10
 
11
  /// A named struct which stores the redis Connection url address to which the client will
12
  /// connect to.
13
+ #[derive(Clone)]
14
  pub struct RedisCache {
15
+ /// It stores a pool of connections ready to be used.
16
+ connection_pool: Vec<ConnectionManager>,
17
+ /// It stores the size of the connection pool (in other words the number of
18
+ /// connections that should be stored in the pool).
19
+ pool_size: u8,
20
+ /// It stores the index of which connection is being used at the moment.
21
+ current_connection: u8,
22
  }
23
 
24
  impl RedisCache {
 
26
  ///
27
  /// # Arguments
28
  ///
29
+ /// * `redis_connection_url` - It takes the redis Connection url address.
30
+ /// * `pool_size` - It takes the size of the connection pool (in other words the number of
31
+ /// connections that should be stored in the pool).
32
+ pub async fn new(
33
+ redis_connection_url: &str,
34
+ pool_size: u8,
35
+ ) -> Result<Self, Box<dyn std::error::Error>> {
36
  let client = Client::open(redis_connection_url)?;
37
+ let mut tasks: Vec<_> = Vec::new();
38
+
39
+ for _ in 0..pool_size {
40
+ tasks.push(client.get_tokio_connection_manager());
41
+ }
42
+
43
+ let redis_cache = RedisCache {
44
+ connection_pool: try_join_all(tasks).await?,
45
+ pool_size,
46
+ current_connection: Default::default(),
47
+ };
48
  Ok(redis_cache)
49
  }
50
 
 
53
  /// # Arguments
54
  ///
55
  /// * `url` - It takes an url as string.
56
+ fn hash_url(&self, url: &str) -> String {
57
  format!("{:?}", compute(url))
58
  }
59
 
 
62
  /// # Arguments
63
  ///
64
  /// * `url` - It takes an url as a string.
65
+ pub async fn cached_json(&mut self, url: &str) -> Result<String, Report<PoolError>> {
66
+ self.current_connection = Default::default();
67
+ let hashed_url_string: &str = &self.hash_url(url);
68
+
69
+ let mut result: Result<String, RedisError> = self.connection_pool
70
+ [self.current_connection as usize]
71
+ .get(hashed_url_string)
72
+ .await;
73
+
74
+ // Code to check whether the current connection being used is dropped with connection error
75
+ // or not. if it drops with the connection error then the current connection is replaced
76
+ // with a new connection from the pool which is then used to run the redis command then
77
+ // that connection is also checked whether it is dropped or not if it is not then the
78
+ // result is passed as a `Result` or else the same process repeats again and if all of the
79
+ // connections in the pool result in connection drop error then a custom pool error is
80
+ // returned.
81
+ loop {
82
+ match result {
83
+ Err(error) => match error.is_connection_dropped() {
84
+ true => {
85
+ self.current_connection += 1;
86
+ if self.current_connection == self.pool_size {
87
+ return Err(Report::new(
88
+ PoolError::PoolExhaustionWithConnectionDropError,
89
+ ));
90
+ }
91
+ result = self.connection_pool[self.current_connection as usize]
92
+ .get(hashed_url_string)
93
+ .await;
94
+ continue;
95
+ }
96
+ false => return Err(Report::new(PoolError::RedisError(error))),
97
+ },
98
+ Ok(res) => return Ok(res),
99
+ }
100
+ }
101
  }
102
 
103
  /// A function which caches the results by using the hashed `url` as the key and
 
108
  ///
109
  /// * `json_results` - It takes the json results string as an argument.
110
  /// * `url` - It takes the url as a String.
111
+ pub async fn cache_results(
112
  &mut self,
113
+ json_results: &str,
114
  url: &str,
115
+ ) -> Result<(), Report<PoolError>> {
116
+ self.current_connection = Default::default();
117
+ let hashed_url_string: &str = &self.hash_url(url);
 
 
118
 
119
+ let mut result: Result<(), RedisError> = self.connection_pool
120
+ [self.current_connection as usize]
121
+ .set_ex(hashed_url_string, json_results, 60)
122
+ .await;
123
 
124
+ // Code to check whether the current connection being used is dropped with connection error
125
+ // or not. if it drops with the connection error then the current connection is replaced
126
+ // with a new connection from the pool which is then used to run the redis command then
127
+ // that connection is also checked whether it is dropped or not if it is not then the
128
+ // result is passed as a `Result` or else the same process repeats again and if all of the
129
+ // connections in the pool result in connection drop error then a custom pool error is
130
+ // returned.
131
+ loop {
132
+ match result {
133
+ Err(error) => match error.is_connection_dropped() {
134
+ true => {
135
+ self.current_connection += 1;
136
+ if self.current_connection == self.pool_size {
137
+ return Err(Report::new(
138
+ PoolError::PoolExhaustionWithConnectionDropError,
139
+ ));
140
+ }
141
+ result = self.connection_pool[self.current_connection as usize]
142
+ .set_ex(hashed_url_string, json_results, 60)
143
+ .await;
144
+ continue;
145
+ }
146
+ false => return Err(Report::new(PoolError::RedisError(error))),
147
+ },
148
+ Ok(_) => return Ok(()),
149
+ }
150
+ }
151
  }
152
  }
src/cache/error.rs ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! This module provides the error enum to handle different errors associated while requesting data from
2
+ //! the redis server using an async connection pool.
3
+ use std::fmt;
4
+
5
+ use redis::RedisError;
6
+
7
+ /// A custom error type used for handling redis async pool associated errors.
8
+ #[derive(Debug)]
9
+ pub enum PoolError {
10
+ /// This variant handles all errors related to `RedisError`,
11
+ RedisError(RedisError),
12
+ /// This variant handles the errors which occurs when all the connections
13
+ /// in the connection pool return a connection dropped redis error.
14
+ PoolExhaustionWithConnectionDropError,
15
+ }
16
+
17
+ impl fmt::Display for PoolError {
18
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
19
+ match self {
20
+ PoolError::RedisError(redis_error) => {
21
+ if let Some(detail) = redis_error.detail() {
22
+ write!(f, "{}", detail)
23
+ } else {
24
+ write!(f, "")
25
+ }
26
+ }
27
+ PoolError::PoolExhaustionWithConnectionDropError => {
28
+ write!(
29
+ f,
30
+ "Error all connections from the pool dropped with connection error"
31
+ )
32
+ }
33
+ }
34
+ }
35
+ }
36
+
37
+ impl error_stack::Context for PoolError {}
src/cache/mod.rs CHANGED
@@ -2,3 +2,4 @@
2
  //! results fetched and aggregated from the upstream search engines in a json format.
3
 
4
  pub mod cacher;
 
 
2
  //! results fetched and aggregated from the upstream search engines in a json format.
3
 
4
  pub mod cacher;
5
+ pub mod error;
src/config/parser.rs CHANGED
@@ -3,9 +3,9 @@
3
 
4
  use crate::handler::paths::{file_path, FileType};
5
 
6
- use crate::models::parser_models::Style;
7
  use log::LevelFilter;
8
- use rlua::Lua;
9
  use std::{collections::HashMap, fs, thread::available_parallelism};
10
 
11
  /// A named struct which stores the parsed config file options.
@@ -32,14 +32,11 @@ pub struct Config {
32
  pub request_timeout: u8,
33
  /// It stores the number of threads which controls the app will use to run.
34
  pub threads: u8,
35
- }
36
-
37
- /// Configuration options for the aggregator.
38
- #[derive(Clone)]
39
- pub struct AggregatorConfig {
40
- /// It stores the option to whether enable or disable random delays between
41
- /// requests.
42
- pub random_delay: bool,
43
  }
44
 
45
  impl Config {
@@ -57,53 +54,70 @@ impl Config {
57
  /// or io error if the config.lua file doesn't exists otherwise it returns a newly constructed
58
  /// Config struct with all the parsed config options from the parsed config file.
59
  pub fn parse(logging_initialized: bool) -> Result<Self, Box<dyn std::error::Error>> {
60
- Lua::new().context(|context| -> Result<Self, Box<dyn std::error::Error>> {
61
- let globals = context.globals();
62
 
63
- context
64
- .load(&fs::read_to_string(file_path(FileType::Config)?)?)
65
- .exec()?;
66
 
67
- let parsed_threads: u8 = globals.get::<_, u8>("threads")?;
68
 
69
- let debug: bool = globals.get::<_, bool>("debug")?;
70
- let logging:bool= globals.get::<_, bool>("logging")?;
71
 
72
- if !logging_initialized {
73
- set_logging_level(debug, logging);
74
- }
 
 
 
 
 
 
 
 
 
 
 
75
 
76
- let threads: u8 = if parsed_threads == 0 {
77
- let total_num_of_threads: usize = available_parallelism()?.get() / 2;
78
- log::error!("Config Error: The value of `threads` option should be a non zero positive integer");
79
- log::error!("Falling back to using {} threads", total_num_of_threads);
80
- total_num_of_threads as u8
81
- } else {
82
- parsed_threads
83
- };
 
 
 
84
 
85
- Ok(Config {
86
- port: globals.get::<_, u16>("port")?,
87
- binding_ip: globals.get::<_, String>("binding_ip")?,
88
- style: Style::new(
89
- globals.get::<_, String>("theme")?,
90
- globals.get::<_, String>("colorscheme")?,
91
- ),
92
- redis_url: globals.get::<_, String>("redis_url")?,
93
- aggregator: AggregatorConfig {
94
- random_delay: globals.get::<_, bool>("production_use")?,
95
- },
96
- logging,
97
- debug,
98
- upstream_search_engines: globals
99
- .get::<_, HashMap<String, bool>>("upstream_search_engines")?
100
- .into_iter()
101
- .filter_map(|(key, value)| value.then_some(key))
102
- .filter_map(|engine| crate::models::engine_models::EngineHandler::new(&engine))
103
- .collect(),
104
- request_timeout: globals.get::<_, u8>("request_timeout")?,
105
- threads,
106
- })
 
 
 
 
107
  })
108
  }
109
  }
 
3
 
4
  use crate::handler::paths::{file_path, FileType};
5
 
6
+ use crate::models::parser_models::{AggregatorConfig, RateLimiter, Style};
7
  use log::LevelFilter;
8
+ use mlua::Lua;
9
  use std::{collections::HashMap, fs, thread::available_parallelism};
10
 
11
  /// A named struct which stores the parsed config file options.
 
32
  pub request_timeout: u8,
33
  /// It stores the number of threads which controls the app will use to run.
34
  pub threads: u8,
35
+ /// It stores configuration options for the ratelimiting middleware.
36
+ pub rate_limiter: RateLimiter,
37
+ /// It stores the level of safe search to be used for restricting content in the
38
+ /// search results.
39
+ pub safe_search: u8,
 
 
 
40
  }
41
 
42
  impl Config {
 
54
  /// or io error if the config.lua file doesn't exists otherwise it returns a newly constructed
55
  /// Config struct with all the parsed config options from the parsed config file.
56
  pub fn parse(logging_initialized: bool) -> Result<Self, Box<dyn std::error::Error>> {
57
+ let lua = Lua::new();
58
+ let globals = lua.globals();
59
 
60
+ lua.load(&fs::read_to_string(file_path(FileType::Config)?)?)
61
+ .exec()?;
 
62
 
63
+ let parsed_threads: u8 = globals.get::<_, u8>("threads")?;
64
 
65
+ let debug: bool = globals.get::<_, bool>("debug")?;
66
+ let logging: bool = globals.get::<_, bool>("logging")?;
67
 
68
+ if !logging_initialized {
69
+ set_logging_level(debug, logging);
70
+ }
71
+
72
+ let threads: u8 = if parsed_threads == 0 {
73
+ let total_num_of_threads: usize = available_parallelism()?.get() / 2;
74
+ log::error!(
75
+ "Config Error: The value of `threads` option should be a non zero positive integer"
76
+ );
77
+ log::error!("Falling back to using {} threads", total_num_of_threads);
78
+ total_num_of_threads as u8
79
+ } else {
80
+ parsed_threads
81
+ };
82
 
83
+ let rate_limiter = globals.get::<_, HashMap<String, u8>>("rate_limiter")?;
84
+
85
+ let parsed_safe_search: u8 = globals.get::<_, u8>("safe_search")?;
86
+ let safe_search: u8 = match parsed_safe_search {
87
+ 0..=4 => parsed_safe_search,
88
+ _ => {
89
+ log::error!("Config Error: The value of `safe_search` option should be a non zero positive integer from 0 to 4.");
90
+ log::error!("Falling back to using the value `1` for the option");
91
+ 1
92
+ }
93
+ };
94
 
95
+ Ok(Config {
96
+ port: globals.get::<_, u16>("port")?,
97
+ binding_ip: globals.get::<_, String>("binding_ip")?,
98
+ style: Style::new(
99
+ globals.get::<_, String>("theme")?,
100
+ globals.get::<_, String>("colorscheme")?,
101
+ ),
102
+ redis_url: globals.get::<_, String>("redis_url")?,
103
+ aggregator: AggregatorConfig {
104
+ random_delay: globals.get::<_, bool>("production_use")?,
105
+ },
106
+ logging,
107
+ debug,
108
+ upstream_search_engines: globals
109
+ .get::<_, HashMap<String, bool>>("upstream_search_engines")?
110
+ .into_iter()
111
+ .filter_map(|(key, value)| value.then_some(key))
112
+ .filter_map(|engine| crate::models::engine_models::EngineHandler::new(&engine))
113
+ .collect(),
114
+ request_timeout: globals.get::<_, u8>("request_timeout")?,
115
+ threads,
116
+ rate_limiter: RateLimiter {
117
+ number_of_requests: rate_limiter["number_of_requests"],
118
+ time_limit: rate_limiter["time_limit"],
119
+ },
120
+ safe_search,
121
  })
122
  }
123
  }
src/engines/duckduckgo.rs CHANGED
@@ -4,14 +4,14 @@
4
 
5
  use std::collections::HashMap;
6
 
7
- use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT};
8
  use scraper::{Html, Selector};
9
 
10
  use crate::models::aggregation_models::SearchResult;
11
 
12
  use crate::models::engine_models::{EngineError, SearchEngine};
13
 
14
- use error_stack::{IntoReport, Report, Result, ResultExt};
15
 
16
  /// A new DuckDuckGo engine type defined in-order to implement the `SearchEngine` trait which allows to
17
  /// reduce code duplication as well as allows to create vector of different search engines easily.
@@ -21,10 +21,11 @@ pub struct DuckDuckGo;
21
  impl SearchEngine for DuckDuckGo {
22
  async fn results(
23
  &self,
24
- query: String,
25
  page: u32,
26
- user_agent: String,
27
  request_timeout: u8,
 
28
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
29
  // Page number can be missing or empty string and so appropriate handling is required
30
  // so that upstream server recieves valid page number.
@@ -43,38 +44,19 @@ impl SearchEngine for DuckDuckGo {
43
  };
44
 
45
  // initializing HeaderMap and adding appropriate headers.
46
- let mut header_map = HeaderMap::new();
47
- header_map.insert(
48
- USER_AGENT,
49
- user_agent
50
- .parse()
51
- .into_report()
52
- .change_context(EngineError::UnexpectedError)?,
53
- );
54
- header_map.insert(
55
- REFERER,
56
- "https://google.com/"
57
- .parse()
58
- .into_report()
59
- .change_context(EngineError::UnexpectedError)?,
60
- );
61
- header_map.insert(
62
- CONTENT_TYPE,
63
- "application/x-www-form-urlencoded"
64
- .parse()
65
- .into_report()
66
- .change_context(EngineError::UnexpectedError)?,
67
- );
68
- header_map.insert(
69
- COOKIE,
70
- "kl=wt-wt"
71
- .parse()
72
- .into_report()
73
- .change_context(EngineError::UnexpectedError)?,
74
- );
75
 
76
  let document: Html = Html::parse_document(
77
- &DuckDuckGo::fetch_html_from_upstream(self, url, header_map, request_timeout).await?,
78
  );
79
 
80
  let no_result: Selector = Selector::parse(".no-results")
@@ -108,8 +90,7 @@ impl SearchEngine for DuckDuckGo {
108
  .next()
109
  .unwrap()
110
  .inner_html()
111
- .trim()
112
- .to_string(),
113
  format!(
114
  "https://{}",
115
  result
@@ -118,15 +99,15 @@ impl SearchEngine for DuckDuckGo {
118
  .unwrap()
119
  .inner_html()
120
  .trim()
121
- ),
 
122
  result
123
  .select(&result_desc)
124
  .next()
125
  .unwrap()
126
  .inner_html()
127
- .trim()
128
- .to_string(),
129
- vec!["duckduckgo".to_string()],
130
  )
131
  })
132
  .map(|search_result| (search_result.url.clone(), search_result))
 
4
 
5
  use std::collections::HashMap;
6
 
7
+ use reqwest::header::HeaderMap;
8
  use scraper::{Html, Selector};
9
 
10
  use crate::models::aggregation_models::SearchResult;
11
 
12
  use crate::models::engine_models::{EngineError, SearchEngine};
13
 
14
+ use error_stack::{Report, Result, ResultExt};
15
 
16
  /// A new DuckDuckGo engine type defined in-order to implement the `SearchEngine` trait which allows to
17
  /// reduce code duplication as well as allows to create vector of different search engines easily.
 
21
  impl SearchEngine for DuckDuckGo {
22
  async fn results(
23
  &self,
24
+ query: &str,
25
  page: u32,
26
+ user_agent: &str,
27
  request_timeout: u8,
28
+ _safe_search: u8,
29
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
30
  // Page number can be missing or empty string and so appropriate handling is required
31
  // so that upstream server recieves valid page number.
 
44
  };
45
 
46
  // initializing HeaderMap and adding appropriate headers.
47
+ let header_map = HeaderMap::try_from(&HashMap::from([
48
+ ("USER_AGENT".to_string(), user_agent.to_string()),
49
+ ("REFERER".to_string(), "https://google.com/".to_string()),
50
+ (
51
+ "CONTENT_TYPE".to_string(),
52
+ "application/x-www-form-urlencoded".to_string(),
53
+ ),
54
+ ("COOKIE".to_string(), "kl=wt-wt".to_string()),
55
+ ]))
56
+ .change_context(EngineError::UnexpectedError)?;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
 
58
  let document: Html = Html::parse_document(
59
+ &DuckDuckGo::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?,
60
  );
61
 
62
  let no_result: Selector = Selector::parse(".no-results")
 
90
  .next()
91
  .unwrap()
92
  .inner_html()
93
+ .trim(),
 
94
  format!(
95
  "https://{}",
96
  result
 
99
  .unwrap()
100
  .inner_html()
101
  .trim()
102
+ )
103
+ .as_str(),
104
  result
105
  .select(&result_desc)
106
  .next()
107
  .unwrap()
108
  .inner_html()
109
+ .trim(),
110
+ &["duckduckgo"],
 
111
  )
112
  })
113
  .map(|search_result| (search_result.url.clone(), search_result))
src/engines/searx.rs CHANGED
@@ -2,14 +2,13 @@
2
  //! by querying the upstream searx search engine instance with user provided query and with a page
3
  //! number if provided.
4
 
5
- use reqwest::header::{HeaderMap, CONTENT_TYPE, COOKIE, REFERER, USER_AGENT};
6
  use scraper::{Html, Selector};
7
  use std::collections::HashMap;
8
 
9
  use crate::models::aggregation_models::SearchResult;
10
-
11
  use crate::models::engine_models::{EngineError, SearchEngine};
12
- use error_stack::{IntoReport, Report, Result, ResultExt};
13
 
14
  /// A new Searx engine type defined in-order to implement the `SearchEngine` trait which allows to
15
  /// reduce code duplication as well as allows to create vector of different search engines easily.
@@ -19,45 +18,38 @@ pub struct Searx;
19
  impl SearchEngine for Searx {
20
  async fn results(
21
  &self,
22
- query: String,
23
  page: u32,
24
- user_agent: String,
25
  request_timeout: u8,
 
26
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
27
  // Page number can be missing or empty string and so appropriate handling is required
28
  // so that upstream server recieves valid page number.
 
 
 
 
29
  let url: String = match page {
30
- 0 | 1 => format!("https://searx.work/search?q={query}&pageno=1"),
31
- _ => format!("https://searx.work/search?q={query}&pageno={page}"),
 
 
 
 
32
  };
33
 
34
  // initializing headers and adding appropriate headers.
35
- let mut header_map = HeaderMap::new();
36
- header_map.insert(
37
- USER_AGENT,
38
- user_agent
39
- .parse()
40
- .into_report()
41
- .change_context(EngineError::UnexpectedError)?,
42
- );
43
- header_map.insert(
44
- REFERER,
45
- "https://google.com/"
46
- .parse()
47
- .into_report()
48
- .change_context(EngineError::UnexpectedError)?,
49
- );
50
- header_map.insert(
51
- CONTENT_TYPE,
52
- "application/x-www-form-urlencoded"
53
- .parse()
54
- .into_report()
55
- .change_context(EngineError::UnexpectedError)?,
56
- );
57
- header_map.insert(COOKIE, "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".parse().into_report().change_context(EngineError::UnexpectedError)?);
58
 
59
  let document: Html = Html::parse_document(
60
- &Searx::fetch_html_from_upstream(self, url, header_map, request_timeout).await?,
61
  );
62
 
63
  let no_result: Selector = Selector::parse("#urls>.dialog-error>p")
@@ -98,24 +90,21 @@ impl SearchEngine for Searx {
98
  .next()
99
  .unwrap()
100
  .inner_html()
101
- .trim()
102
- .to_string(),
103
  result
104
  .select(&result_url)
105
  .next()
106
  .unwrap()
107
  .value()
108
  .attr("href")
109
- .unwrap()
110
- .to_string(),
111
  result
112
  .select(&result_desc)
113
  .next()
114
  .unwrap()
115
  .inner_html()
116
- .trim()
117
- .to_string(),
118
- vec!["searx".to_string()],
119
  )
120
  })
121
  .map(|search_result| (search_result.url.clone(), search_result))
 
2
  //! by querying the upstream searx search engine instance with user provided query and with a page
3
  //! number if provided.
4
 
5
+ use reqwest::header::HeaderMap;
6
  use scraper::{Html, Selector};
7
  use std::collections::HashMap;
8
 
9
  use crate::models::aggregation_models::SearchResult;
 
10
  use crate::models::engine_models::{EngineError, SearchEngine};
11
+ use error_stack::{Report, Result, ResultExt};
12
 
13
  /// A new Searx engine type defined in-order to implement the `SearchEngine` trait which allows to
14
  /// reduce code duplication as well as allows to create vector of different search engines easily.
 
18
  impl SearchEngine for Searx {
19
  async fn results(
20
  &self,
21
+ query: &str,
22
  page: u32,
23
+ user_agent: &str,
24
  request_timeout: u8,
25
+ mut safe_search: u8,
26
  ) -> Result<HashMap<String, SearchResult>, EngineError> {
27
  // Page number can be missing or empty string and so appropriate handling is required
28
  // so that upstream server recieves valid page number.
29
+ if safe_search == 3 {
30
+ safe_search = 2;
31
+ };
32
+
33
  let url: String = match page {
34
+ 0 | 1 => {
35
+ format!("https://searx.work/search?q={query}&pageno=1&safesearch={safe_search}")
36
+ }
37
+ _ => format!(
38
+ "https://searx.work/search?q={query}&pageno={page}&safesearch={safe_search}"
39
+ ),
40
  };
41
 
42
  // initializing headers and adding appropriate headers.
43
+ let header_map = HeaderMap::try_from(&HashMap::from([
44
+ ("USER_AGENT".to_string(), user_agent.to_string()),
45
+ ("REFERER".to_string(), "https://google.com/".to_string()),
46
+ ("CONTENT_TYPE".to_string(), "application/x-www-form-urlencoded".to_string()),
47
+ ("COOKIE".to_string(), "categories=general; language=auto; locale=en; autocomplete=duckduckgo; image_proxy=1; method=POST; safesearch=2; theme=simple; results_on_new_tab=1; doi_resolver=oadoi.org; simple_style=auto; center_alignment=1; query_in_title=1; infinite_scroll=0; disabled_engines=; enabled_engines=\"archive is__general\\054yep__general\\054curlie__general\\054currency__general\\054ddg definitions__general\\054wikidata__general\\054duckduckgo__general\\054tineye__general\\054lingva__general\\054startpage__general\\054yahoo__general\\054wiby__general\\054marginalia__general\\054alexandria__general\\054wikibooks__general\\054wikiquote__general\\054wikisource__general\\054wikiversity__general\\054wikivoyage__general\\054dictzone__general\\054seznam__general\\054mojeek__general\\054naver__general\\054wikimini__general\\054brave__general\\054petalsearch__general\\054goo__general\"; disabled_plugins=; enabled_plugins=\"searx.plugins.hostname_replace\\054searx.plugins.oa_doi_rewrite\\054searx.plugins.vim_hotkeys\"; tokens=; maintab=on; enginetab=on".to_string())
48
+ ]))
49
+ .change_context(EngineError::UnexpectedError)?;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
50
 
51
  let document: Html = Html::parse_document(
52
+ &Searx::fetch_html_from_upstream(self, &url, header_map, request_timeout).await?,
53
  );
54
 
55
  let no_result: Selector = Selector::parse("#urls>.dialog-error>p")
 
90
  .next()
91
  .unwrap()
92
  .inner_html()
93
+ .trim(),
 
94
  result
95
  .select(&result_url)
96
  .next()
97
  .unwrap()
98
  .value()
99
  .attr("href")
100
+ .unwrap(),
 
101
  result
102
  .select(&result_desc)
103
  .next()
104
  .unwrap()
105
  .inner_html()
106
+ .trim(),
107
+ &["searx"],
 
108
  )
109
  })
110
  .map(|search_result| (search_result.url.clone(), search_result))
src/handler/paths.rs CHANGED
@@ -4,6 +4,7 @@
4
  use std::collections::HashMap;
5
  use std::io::Error;
6
  use std::path::Path;
 
7
 
8
  // ------- Constants --------
9
  /// The constant holding the name of the theme folder.
@@ -31,57 +32,7 @@ pub enum FileType {
31
  }
32
 
33
  /// A static variable which stores the different filesystem paths for various file/folder types.
34
- static FILE_PATHS_FOR_DIFF_FILE_TYPES: once_cell::sync::Lazy<HashMap<FileType, Vec<String>>> =
35
- once_cell::sync::Lazy::new(|| {
36
- HashMap::from([
37
- (
38
- FileType::Config,
39
- vec![
40
- format!(
41
- "{}/.config/{}/{}",
42
- std::env::var("HOME").unwrap(),
43
- COMMON_DIRECTORY_NAME,
44
- CONFIG_FILE_NAME
45
- ),
46
- format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME),
47
- format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME),
48
- ],
49
- ),
50
- (
51
- FileType::Theme,
52
- vec![
53
- format!("/opt/websurfx/{}/", PUBLIC_DIRECTORY_NAME),
54
- format!("./{}/", PUBLIC_DIRECTORY_NAME),
55
- ],
56
- ),
57
- (
58
- FileType::AllowList,
59
- vec![
60
- format!(
61
- "{}/.config/{}/{}",
62
- std::env::var("HOME").unwrap(),
63
- COMMON_DIRECTORY_NAME,
64
- ALLOWLIST_FILE_NAME
65
- ),
66
- format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME),
67
- format!("./{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME),
68
- ],
69
- ),
70
- (
71
- FileType::BlockList,
72
- vec![
73
- format!(
74
- "{}/.config/{}/{}",
75
- std::env::var("HOME").unwrap(),
76
- COMMON_DIRECTORY_NAME,
77
- BLOCKLIST_FILE_NAME
78
- ),
79
- format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME),
80
- format!("./{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME),
81
- ],
82
- ),
83
- ])
84
- });
85
 
86
  /// A function which returns an appropriate path for thr provided file type by checking if the path
87
  /// for the given file type exists on that path.
@@ -99,11 +50,64 @@ static FILE_PATHS_FOR_DIFF_FILE_TYPES: once_cell::sync::Lazy<HashMap<FileType, V
99
  /// 1. `/opt/websurfx` if it not present here then it fallbacks to the next one (2)
100
  /// 2. Under project folder ( or codebase in other words) if it is not present
101
  /// here then it returns an error as mentioned above.
102
- pub fn file_path(file_type: FileType) -> Result<String, Error> {
103
- let file_path = FILE_PATHS_FOR_DIFF_FILE_TYPES.get(&file_type).unwrap();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
104
  for (idx, _) in file_path.iter().enumerate() {
105
  if Path::new(file_path[idx].as_str()).exists() {
106
- return Ok(file_path[idx].clone());
107
  }
108
  }
109
 
 
4
  use std::collections::HashMap;
5
  use std::io::Error;
6
  use std::path::Path;
7
+ use std::sync::OnceLock;
8
 
9
  // ------- Constants --------
10
  /// The constant holding the name of the theme folder.
 
32
  }
33
 
34
  /// A static variable which stores the different filesystem paths for various file/folder types.
35
+ static FILE_PATHS_FOR_DIFF_FILE_TYPES: OnceLock<HashMap<FileType, Vec<String>>> = OnceLock::new();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
36
 
37
  /// A function which returns an appropriate path for thr provided file type by checking if the path
38
  /// for the given file type exists on that path.
 
50
  /// 1. `/opt/websurfx` if it not present here then it fallbacks to the next one (2)
51
  /// 2. Under project folder ( or codebase in other words) if it is not present
52
  /// here then it returns an error as mentioned above.
53
+ pub fn file_path(file_type: FileType) -> Result<&'static str, Error> {
54
+ let file_path: &Vec<String> = FILE_PATHS_FOR_DIFF_FILE_TYPES
55
+ .get_or_init(|| {
56
+ HashMap::from([
57
+ (
58
+ FileType::Config,
59
+ vec![
60
+ format!(
61
+ "{}/.config/{}/{}",
62
+ std::env::var("HOME").unwrap(),
63
+ COMMON_DIRECTORY_NAME,
64
+ CONFIG_FILE_NAME
65
+ ),
66
+ format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME),
67
+ format!("./{}/{}", COMMON_DIRECTORY_NAME, CONFIG_FILE_NAME),
68
+ ],
69
+ ),
70
+ (
71
+ FileType::Theme,
72
+ vec![
73
+ format!("/opt/websurfx/{}/", PUBLIC_DIRECTORY_NAME),
74
+ format!("./{}/", PUBLIC_DIRECTORY_NAME),
75
+ ],
76
+ ),
77
+ (
78
+ FileType::AllowList,
79
+ vec![
80
+ format!(
81
+ "{}/.config/{}/{}",
82
+ std::env::var("HOME").unwrap(),
83
+ COMMON_DIRECTORY_NAME,
84
+ ALLOWLIST_FILE_NAME
85
+ ),
86
+ format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME),
87
+ format!("./{}/{}", COMMON_DIRECTORY_NAME, ALLOWLIST_FILE_NAME),
88
+ ],
89
+ ),
90
+ (
91
+ FileType::BlockList,
92
+ vec![
93
+ format!(
94
+ "{}/.config/{}/{}",
95
+ std::env::var("HOME").unwrap(),
96
+ COMMON_DIRECTORY_NAME,
97
+ BLOCKLIST_FILE_NAME
98
+ ),
99
+ format!("/etc/xdg/{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME),
100
+ format!("./{}/{}", COMMON_DIRECTORY_NAME, BLOCKLIST_FILE_NAME),
101
+ ],
102
+ ),
103
+ ])
104
+ })
105
+ .get(&file_type)
106
+ .unwrap();
107
+
108
  for (idx, _) in file_path.iter().enumerate() {
109
  if Path::new(file_path[idx].as_str()).exists() {
110
+ return Ok(std::mem::take(&mut &*file_path[idx]));
111
  }
112
  }
113
 
src/lib.rs CHANGED
@@ -19,6 +19,7 @@ use crate::server::router;
19
 
20
  use actix_cors::Cors;
21
  use actix_files as fs;
 
22
  use actix_web::{dev::Server, http::header, middleware::Logger, web, App, HttpServer};
23
  use config::parser::Config;
24
  use handlebars::Handlebars;
@@ -47,7 +48,7 @@ use handler::paths::{file_path, FileType};
47
  pub fn run(listener: TcpListener, config: Config) -> std::io::Result<Server> {
48
  let mut handlebars: Handlebars<'_> = Handlebars::new();
49
 
50
- let public_folder_path: String = file_path(FileType::Theme)?;
51
 
52
  handlebars
53
  .register_templates_directory(".html", format!("{}/templates", public_folder_path))
@@ -69,10 +70,17 @@ pub fn run(listener: TcpListener, config: Config) -> std::io::Result<Server> {
69
  ]);
70
 
71
  App::new()
 
72
  .app_data(handlebars_ref.clone())
73
  .app_data(web::Data::new(config.clone()))
74
  .wrap(cors)
75
- .wrap(Logger::default()) // added logging middleware for logging.
 
 
 
 
 
 
76
  // Serve images and static files (css and js files).
77
  .service(
78
  fs::Files::new("/static", format!("{}/static", public_folder_path))
 
19
 
20
  use actix_cors::Cors;
21
  use actix_files as fs;
22
+ use actix_governor::{Governor, GovernorConfigBuilder};
23
  use actix_web::{dev::Server, http::header, middleware::Logger, web, App, HttpServer};
24
  use config::parser::Config;
25
  use handlebars::Handlebars;
 
48
  pub fn run(listener: TcpListener, config: Config) -> std::io::Result<Server> {
49
  let mut handlebars: Handlebars<'_> = Handlebars::new();
50
 
51
+ let public_folder_path: &str = file_path(FileType::Theme)?;
52
 
53
  handlebars
54
  .register_templates_directory(".html", format!("{}/templates", public_folder_path))
 
70
  ]);
71
 
72
  App::new()
73
+ .wrap(Logger::default()) // added logging middleware for logging.
74
  .app_data(handlebars_ref.clone())
75
  .app_data(web::Data::new(config.clone()))
76
  .wrap(cors)
77
+ .wrap(Governor::new(
78
+ &GovernorConfigBuilder::default()
79
+ .per_second(config.rate_limiter.time_limit as u64)
80
+ .burst_size(config.rate_limiter.number_of_requests as u32)
81
+ .finish()
82
+ .unwrap(),
83
+ ))
84
  // Serve images and static files (css and js files).
85
  .service(
86
  fs::Files::new("/static", format!("{}/static", public_folder_path))
src/models/aggregation_models.rs CHANGED
@@ -2,6 +2,7 @@
2
  //! data scraped from the upstream search engines.
3
 
4
  use serde::{Deserialize, Serialize};
 
5
 
6
  use super::{engine_models::EngineError, parser_models::Style};
7
 
@@ -19,7 +20,7 @@ pub struct SearchResult {
19
  /// The description of the search result.
20
  pub description: String,
21
  /// The names of the upstream engines from which this results were provided.
22
- pub engine: Vec<String>,
23
  }
24
 
25
  impl SearchResult {
@@ -32,12 +33,12 @@ impl SearchResult {
32
  /// (href url in html in simple words).
33
  /// * `description` - The description of the search result.
34
  /// * `engine` - The names of the upstream engines from which this results were provided.
35
- pub fn new(title: String, url: String, description: String, engine: Vec<String>) -> Self {
36
  SearchResult {
37
- title,
38
- url,
39
- description,
40
- engine,
41
  }
42
  }
43
 
@@ -46,8 +47,8 @@ impl SearchResult {
46
  /// # Arguments
47
  ///
48
  /// * `engine` - Takes an engine name provided as a String.
49
- pub fn add_engines(&mut self, engine: String) {
50
- self.engine.push(engine)
51
  }
52
 
53
  /// A function which returns the engine name stored from the struct as a string.
@@ -55,13 +56,13 @@ impl SearchResult {
55
  /// # Returns
56
  ///
57
  /// An engine name stored as a string from the struct.
58
- pub fn engine(self) -> String {
59
- self.engine.get(0).unwrap().to_string()
60
  }
61
  }
62
 
63
  /// A named struct that stores the error info related to the upstream search engines.
64
- #[derive(Serialize, Deserialize)]
65
  pub struct EngineErrorInfo {
66
  /// It stores the error type which occured while fetching the result from a particular search
67
  /// engine.
@@ -81,18 +82,18 @@ impl EngineErrorInfo {
81
  /// * `error` - It takes the error type which occured while fetching the result from a particular
82
  /// search engine.
83
  /// * `engine` - It takes the name of the engine that failed to provide the requested search results.
84
- pub fn new(error: &EngineError, engine: String) -> Self {
85
  Self {
86
  error: match error {
87
- EngineError::RequestError => String::from("RequestError"),
88
- EngineError::EmptyResultSet => String::from("EmptyResultSet"),
89
- EngineError::UnexpectedError => String::from("UnexpectedError"),
90
  },
91
- engine,
92
  severity_color: match error {
93
- EngineError::RequestError => String::from("green"),
94
- EngineError::EmptyResultSet => String::from("blue"),
95
- EngineError::UnexpectedError => String::from("red"),
96
  },
97
  }
98
  }
@@ -101,7 +102,7 @@ impl EngineErrorInfo {
101
  /// A named struct to store, serialize, deserialize the all the search results scraped and
102
  /// aggregated from the upstream search engines.
103
  /// `SearchResult` structs.
104
- #[derive(Serialize, Deserialize)]
105
  #[serde(rename_all = "camelCase")]
106
  pub struct SearchResults {
107
  /// Stores the individual serializable `SearchResult` struct into a vector of
@@ -113,6 +114,14 @@ pub struct SearchResults {
113
  /// Stores the information on which engines failed with their engine name
114
  /// and the type of error that caused it.
115
  pub engine_errors_info: Vec<EngineErrorInfo>,
 
 
 
 
 
 
 
 
116
  }
117
 
118
  impl SearchResults {
@@ -124,23 +133,49 @@ impl SearchResults {
124
  /// and stores it into a vector of `SearchResult` structs.
125
  /// * `page_query` - Takes an argument of current page`s search query `q` provided in
126
  /// the search url.
127
- /// * `empty_result_set` - Takes a boolean which indicates that no engines gave a result for the
128
- /// given search query.
129
  pub fn new(
130
  results: Vec<SearchResult>,
131
- page_query: String,
132
- engine_errors_info: Vec<EngineErrorInfo>,
133
  ) -> Self {
134
- SearchResults {
135
  results,
136
- page_query,
137
- style: Style::new("".to_string(), "".to_string()),
138
- engine_errors_info,
 
 
139
  }
140
  }
141
 
142
  /// A setter function to add website style to the return search results.
143
- pub fn add_style(&mut self, style: Style) {
144
- self.style = style;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
145
  }
146
  }
 
2
  //! data scraped from the upstream search engines.
3
 
4
  use serde::{Deserialize, Serialize};
5
+ use smallvec::SmallVec;
6
 
7
  use super::{engine_models::EngineError, parser_models::Style};
8
 
 
20
  /// The description of the search result.
21
  pub description: String,
22
  /// The names of the upstream engines from which this results were provided.
23
+ pub engine: SmallVec<[String; 0]>,
24
  }
25
 
26
  impl SearchResult {
 
33
  /// (href url in html in simple words).
34
  /// * `description` - The description of the search result.
35
  /// * `engine` - The names of the upstream engines from which this results were provided.
36
+ pub fn new(title: &str, url: &str, description: &str, engine: &[&str]) -> Self {
37
  SearchResult {
38
+ title: title.to_owned(),
39
+ url: url.to_owned(),
40
+ description: description.to_owned(),
41
+ engine: engine.iter().map(|name| name.to_string()).collect(),
42
  }
43
  }
44
 
 
47
  /// # Arguments
48
  ///
49
  /// * `engine` - Takes an engine name provided as a String.
50
+ pub fn add_engines(&mut self, engine: &str) {
51
+ self.engine.push(engine.to_owned())
52
  }
53
 
54
  /// A function which returns the engine name stored from the struct as a string.
 
56
  /// # Returns
57
  ///
58
  /// An engine name stored as a string from the struct.
59
+ pub fn engine(&mut self) -> String {
60
+ std::mem::take(&mut self.engine[0])
61
  }
62
  }
63
 
64
  /// A named struct that stores the error info related to the upstream search engines.
65
+ #[derive(Serialize, Deserialize, Clone)]
66
  pub struct EngineErrorInfo {
67
  /// It stores the error type which occured while fetching the result from a particular search
68
  /// engine.
 
82
  /// * `error` - It takes the error type which occured while fetching the result from a particular
83
  /// search engine.
84
  /// * `engine` - It takes the name of the engine that failed to provide the requested search results.
85
+ pub fn new(error: &EngineError, engine: &str) -> Self {
86
  Self {
87
  error: match error {
88
+ EngineError::RequestError => "RequestError".to_owned(),
89
+ EngineError::EmptyResultSet => "EmptyResultSet".to_owned(),
90
+ EngineError::UnexpectedError => "UnexpectedError".to_owned(),
91
  },
92
+ engine: engine.to_owned(),
93
  severity_color: match error {
94
+ EngineError::RequestError => "green".to_owned(),
95
+ EngineError::EmptyResultSet => "blue".to_owned(),
96
+ EngineError::UnexpectedError => "red".to_owned(),
97
  },
98
  }
99
  }
 
102
  /// A named struct to store, serialize, deserialize the all the search results scraped and
103
  /// aggregated from the upstream search engines.
104
  /// `SearchResult` structs.
105
+ #[derive(Serialize, Deserialize, Default)]
106
  #[serde(rename_all = "camelCase")]
107
  pub struct SearchResults {
108
  /// Stores the individual serializable `SearchResult` struct into a vector of
 
114
  /// Stores the information on which engines failed with their engine name
115
  /// and the type of error that caused it.
116
  pub engine_errors_info: Vec<EngineErrorInfo>,
117
+ /// Stores the flag option which holds the check value that the following
118
+ /// search query was disallowed when the safe search level set to 4 and it
119
+ /// was present in the `Blocklist` file.
120
+ pub disallowed: bool,
121
+ /// Stores the flag option which holds the check value that the following
122
+ /// search query was filtered when the safe search level set to 3 and it
123
+ /// was present in the `Blocklist` file.
124
+ pub filtered: bool,
125
  }
126
 
127
  impl SearchResults {
 
133
  /// and stores it into a vector of `SearchResult` structs.
134
  /// * `page_query` - Takes an argument of current page`s search query `q` provided in
135
  /// the search url.
136
+ /// * `engine_errors_info` - Takes an array of structs which contains information regarding
137
+ /// which engines failed with their names, reason and their severity color name.
138
  pub fn new(
139
  results: Vec<SearchResult>,
140
+ page_query: &str,
141
+ engine_errors_info: &[EngineErrorInfo],
142
  ) -> Self {
143
+ Self {
144
  results,
145
+ page_query: page_query.to_owned(),
146
+ style: Style::default(),
147
+ engine_errors_info: engine_errors_info.to_owned(),
148
+ disallowed: Default::default(),
149
+ filtered: Default::default(),
150
  }
151
  }
152
 
153
  /// A setter function to add website style to the return search results.
154
+ pub fn add_style(&mut self, style: &Style) {
155
+ self.style = style.clone();
156
+ }
157
+
158
+ /// A setter function that sets disallowed to true.
159
+ pub fn set_disallowed(&mut self) {
160
+ self.disallowed = true;
161
+ }
162
+
163
+ /// A setter function to set the current page search query.
164
+ pub fn set_page_query(&mut self, page: &str) {
165
+ self.page_query = page.to_owned();
166
+ }
167
+
168
+ /// A setter function that sets the filtered to true.
169
+ pub fn set_filtered(&mut self) {
170
+ self.filtered = true;
171
+ }
172
+
173
+ /// A getter function that gets the value of `engine_errors_info`.
174
+ pub fn engine_errors_info(&mut self) -> Vec<EngineErrorInfo> {
175
+ std::mem::take(&mut self.engine_errors_info)
176
+ }
177
+ /// A getter function that gets the value of `results`.
178
+ pub fn results(&mut self) -> Vec<SearchResult> {
179
+ self.results.clone()
180
  }
181
  }
src/models/engine_models.rs CHANGED
@@ -2,7 +2,7 @@
2
  //! the upstream search engines with the search query provided by the user.
3
 
4
  use super::aggregation_models::SearchResult;
5
- use error_stack::{IntoReport, Result, ResultExt};
6
  use std::{collections::HashMap, fmt, time::Duration};
7
 
8
  /// A custom error type used for handle engine associated errors.
@@ -64,7 +64,7 @@ pub trait SearchEngine: Sync + Send {
64
  /// otherwise it returns a custom `EngineError`.
65
  async fn fetch_html_from_upstream(
66
  &self,
67
- url: String,
68
  header_map: reqwest::header::HeaderMap,
69
  request_timeout: u8,
70
  ) -> Result<String, EngineError> {
@@ -75,11 +75,9 @@ pub trait SearchEngine: Sync + Send {
75
  .headers(header_map) // add spoofed headers to emulate human behavior
76
  .send()
77
  .await
78
- .into_report()
79
  .change_context(EngineError::RequestError)?
80
  .text()
81
  .await
82
- .into_report()
83
  .change_context(EngineError::RequestError)?)
84
  }
85
 
@@ -103,10 +101,11 @@ pub trait SearchEngine: Sync + Send {
103
  /// or HeaderMap fails to initialize.
104
  async fn results(
105
  &self,
106
- query: String,
107
  page: u32,
108
- user_agent: String,
109
  request_timeout: u8,
 
110
  ) -> Result<HashMap<String, SearchResult>, EngineError>;
111
  }
112
 
 
2
  //! the upstream search engines with the search query provided by the user.
3
 
4
  use super::aggregation_models::SearchResult;
5
+ use error_stack::{Result, ResultExt};
6
  use std::{collections::HashMap, fmt, time::Duration};
7
 
8
  /// A custom error type used for handle engine associated errors.
 
64
  /// otherwise it returns a custom `EngineError`.
65
  async fn fetch_html_from_upstream(
66
  &self,
67
+ url: &str,
68
  header_map: reqwest::header::HeaderMap,
69
  request_timeout: u8,
70
  ) -> Result<String, EngineError> {
 
75
  .headers(header_map) // add spoofed headers to emulate human behavior
76
  .send()
77
  .await
 
78
  .change_context(EngineError::RequestError)?
79
  .text()
80
  .await
 
81
  .change_context(EngineError::RequestError)?)
82
  }
83
 
 
101
  /// or HeaderMap fails to initialize.
102
  async fn results(
103
  &self,
104
+ query: &str,
105
  page: u32,
106
+ user_agent: &str,
107
  request_timeout: u8,
108
+ safe_search: u8,
109
  ) -> Result<HashMap<String, SearchResult>, EngineError>;
110
  }
111
 
src/models/parser_models.rs CHANGED
@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
12
  /// order to allow the deserializing the json back to struct in aggregate function in
13
  /// aggregator.rs and create a new struct out of it and then serialize it back to json and pass
14
  /// it to the template files.
15
- #[derive(Serialize, Deserialize, Clone)]
16
  pub struct Style {
17
  /// It stores the parsed theme option used to set a theme for the website.
18
  pub theme: String,
@@ -33,3 +33,20 @@ impl Style {
33
  Style { theme, colorscheme }
34
  }
35
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
12
  /// order to allow the deserializing the json back to struct in aggregate function in
13
  /// aggregator.rs and create a new struct out of it and then serialize it back to json and pass
14
  /// it to the template files.
15
+ #[derive(Serialize, Deserialize, Clone, Default)]
16
  pub struct Style {
17
  /// It stores the parsed theme option used to set a theme for the website.
18
  pub theme: String,
 
33
  Style { theme, colorscheme }
34
  }
35
  }
36
+
37
+ /// Configuration options for the aggregator.
38
+ #[derive(Clone)]
39
+ pub struct AggregatorConfig {
40
+ /// It stores the option to whether enable or disable random delays between
41
+ /// requests.
42
+ pub random_delay: bool,
43
+ }
44
+
45
+ /// Configuration options for the rate limiter middleware.
46
+ #[derive(Clone)]
47
+ pub struct RateLimiter {
48
+ /// The number of request that are allowed within a provided time limit.
49
+ pub number_of_requests: u8,
50
+ /// The time limit in which the quantity of requests that should be accepted.
51
+ pub time_limit: u8,
52
+ }
src/results/aggregator.rs CHANGED
@@ -60,14 +60,15 @@ type FutureVec = Vec<JoinHandle<Result<HashMap<String, SearchResult>, Report<Eng
60
  /// function in either `searx` or `duckduckgo` or both otherwise returns a `SearchResults struct`
61
  /// containing appropriate values.
62
  pub async fn aggregate(
63
- query: String,
64
  page: u32,
65
  random_delay: bool,
66
  debug: bool,
67
- upstream_search_engines: Vec<EngineHandler>,
68
  request_timeout: u8,
 
69
  ) -> Result<SearchResults, Box<dyn std::error::Error>> {
70
- let user_agent: String = random_user_agent();
71
 
72
  // Add a random delay before making the request.
73
  if random_delay || !debug {
@@ -76,19 +77,24 @@ pub async fn aggregate(
76
  tokio::time::sleep(Duration::from_secs(delay_secs)).await;
77
  }
78
 
79
- let mut names: Vec<&str> = vec![];
80
 
81
  // create tasks for upstream result fetching
82
  let mut tasks: FutureVec = FutureVec::new();
83
 
84
  for engine_handler in upstream_search_engines {
85
- let (name, search_engine) = engine_handler.into_name_engine();
86
  names.push(name);
87
- let query: String = query.clone();
88
- let user_agent: String = user_agent.clone();
89
  tasks.push(tokio::spawn(async move {
90
  search_engine
91
- .results(query, page, user_agent.clone(), request_timeout)
 
 
 
 
 
 
92
  .await
93
  }));
94
  }
@@ -106,7 +112,7 @@ pub async fn aggregate(
106
  let mut result_map: HashMap<String, SearchResult> = HashMap::new();
107
  let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new();
108
 
109
- let mut handle_error = |error: Report<EngineError>, engine_name: String| {
110
  log::error!("Engine Error: {:?}", error);
111
  engine_errors_info.push(EngineErrorInfo::new(
112
  error.downcast_ref::<EngineError>().unwrap(),
@@ -116,7 +122,7 @@ pub async fn aggregate(
116
 
117
  for _ in 0..responses.len() {
118
  let response = responses.pop().unwrap();
119
- let engine = names.pop().unwrap().to_string();
120
 
121
  if result_map.is_empty() {
122
  match response {
@@ -124,7 +130,7 @@ pub async fn aggregate(
124
  result_map = results.clone();
125
  }
126
  Err(error) => {
127
- handle_error(error, engine);
128
  }
129
  }
130
  continue;
@@ -136,39 +142,37 @@ pub async fn aggregate(
136
  result_map
137
  .entry(key)
138
  .and_modify(|result| {
139
- result.add_engines(engine.clone());
140
  })
141
  .or_insert_with(|| -> SearchResult { value });
142
  });
143
  }
144
  Err(error) => {
145
- handle_error(error, engine);
146
  }
147
  }
148
  }
149
 
150
- let mut blacklist_map: HashMap<String, SearchResult> = HashMap::new();
151
- filter_with_lists(
152
- &mut result_map,
153
- &mut blacklist_map,
154
- &file_path(FileType::BlockList)?,
155
- )?;
 
156
 
157
- filter_with_lists(
158
- &mut blacklist_map,
159
- &mut result_map,
160
- &file_path(FileType::AllowList)?,
161
- )?;
162
 
163
- drop(blacklist_map);
 
164
 
165
  let results: Vec<SearchResult> = result_map.into_values().collect();
166
 
167
- Ok(SearchResults::new(
168
- results,
169
- query.to_string(),
170
- engine_errors_info,
171
- ))
172
  }
173
 
174
  /// Filters a map of search results using a list of regex patterns.
@@ -190,7 +194,7 @@ pub fn filter_with_lists(
190
  let mut reader = BufReader::new(File::open(file_path)?);
191
 
192
  for line in reader.by_ref().lines() {
193
- let re = Regex::new(&line?)?;
194
 
195
  // Iterate over each search result in the map and check if it matches the regex pattern
196
  for (url, search_result) in map_to_be_filtered.clone().into_iter() {
@@ -199,7 +203,10 @@ pub fn filter_with_lists(
199
  || re.is_match(&search_result.description.to_lowercase())
200
  {
201
  // If the search result matches the regex pattern, move it from the original map to the resultant map
202
- resultant_map.insert(url.clone(), map_to_be_filtered.remove(&url).unwrap());
 
 
 
203
  }
204
  }
205
  }
@@ -210,6 +217,7 @@ pub fn filter_with_lists(
210
  #[cfg(test)]
211
  mod tests {
212
  use super::*;
 
213
  use std::collections::HashMap;
214
  use std::io::Write;
215
  use tempfile::NamedTempFile;
@@ -219,22 +227,22 @@ mod tests {
219
  // Create a map of search results to filter
220
  let mut map_to_be_filtered = HashMap::new();
221
  map_to_be_filtered.insert(
222
- "https://www.example.com".to_string(),
223
  SearchResult {
224
- title: "Example Domain".to_string(),
225
- url: "https://www.example.com".to_string(),
226
  description: "This domain is for use in illustrative examples in documents."
227
- .to_string(),
228
- engine: vec!["Google".to_string(), "Bing".to_string()],
229
  },
230
  );
231
  map_to_be_filtered.insert(
232
- "https://www.rust-lang.org/".to_string(),
233
  SearchResult {
234
- title: "Rust Programming Language".to_string(),
235
- url: "https://www.rust-lang.org/".to_string(),
236
- description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_string(),
237
- engine: vec!["Google".to_string(), "DuckDuckGo".to_string()],
238
  },
239
  );
240
 
@@ -263,22 +271,22 @@ mod tests {
263
  fn test_filter_with_lists_wildcard() -> Result<(), Box<dyn std::error::Error>> {
264
  let mut map_to_be_filtered = HashMap::new();
265
  map_to_be_filtered.insert(
266
- "https://www.example.com".to_string(),
267
  SearchResult {
268
- title: "Example Domain".to_string(),
269
- url: "https://www.example.com".to_string(),
270
  description: "This domain is for use in illustrative examples in documents."
271
- .to_string(),
272
- engine: vec!["Google".to_string(), "Bing".to_string()],
273
  },
274
  );
275
  map_to_be_filtered.insert(
276
- "https://www.rust-lang.org/".to_string(),
277
  SearchResult {
278
- title: "Rust Programming Language".to_string(),
279
- url: "https://www.rust-lang.org/".to_string(),
280
- description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_string(),
281
- engine: vec!["Google".to_string(), "DuckDuckGo".to_string()],
282
  },
283
  );
284
 
@@ -323,13 +331,13 @@ mod tests {
323
  fn test_filter_with_lists_invalid_regex() {
324
  let mut map_to_be_filtered = HashMap::new();
325
  map_to_be_filtered.insert(
326
- "https://www.example.com".to_string(),
327
  SearchResult {
328
- title: "Example Domain".to_string(),
329
- url: "https://www.example.com".to_string(),
330
  description: "This domain is for use in illustrative examples in documents."
331
- .to_string(),
332
- engine: vec!["Google".to_string(), "Bing".to_string()],
333
  },
334
  );
335
 
 
60
  /// function in either `searx` or `duckduckgo` or both otherwise returns a `SearchResults struct`
61
  /// containing appropriate values.
62
  pub async fn aggregate(
63
+ query: &str,
64
  page: u32,
65
  random_delay: bool,
66
  debug: bool,
67
+ upstream_search_engines: &[EngineHandler],
68
  request_timeout: u8,
69
+ safe_search: u8,
70
  ) -> Result<SearchResults, Box<dyn std::error::Error>> {
71
+ let user_agent: &str = random_user_agent();
72
 
73
  // Add a random delay before making the request.
74
  if random_delay || !debug {
 
77
  tokio::time::sleep(Duration::from_secs(delay_secs)).await;
78
  }
79
 
80
+ let mut names: Vec<&str> = Vec::with_capacity(0);
81
 
82
  // create tasks for upstream result fetching
83
  let mut tasks: FutureVec = FutureVec::new();
84
 
85
  for engine_handler in upstream_search_engines {
86
+ let (name, search_engine) = engine_handler.to_owned().into_name_engine();
87
  names.push(name);
88
+ let query: String = query.to_owned();
 
89
  tasks.push(tokio::spawn(async move {
90
  search_engine
91
+ .results(
92
+ &query,
93
+ page,
94
+ user_agent.clone(),
95
+ request_timeout,
96
+ safe_search,
97
+ )
98
  .await
99
  }));
100
  }
 
112
  let mut result_map: HashMap<String, SearchResult> = HashMap::new();
113
  let mut engine_errors_info: Vec<EngineErrorInfo> = Vec::new();
114
 
115
+ let mut handle_error = |error: &Report<EngineError>, engine_name: &'static str| {
116
  log::error!("Engine Error: {:?}", error);
117
  engine_errors_info.push(EngineErrorInfo::new(
118
  error.downcast_ref::<EngineError>().unwrap(),
 
122
 
123
  for _ in 0..responses.len() {
124
  let response = responses.pop().unwrap();
125
+ let engine = names.pop().unwrap();
126
 
127
  if result_map.is_empty() {
128
  match response {
 
130
  result_map = results.clone();
131
  }
132
  Err(error) => {
133
+ handle_error(&error, engine);
134
  }
135
  }
136
  continue;
 
142
  result_map
143
  .entry(key)
144
  .and_modify(|result| {
145
+ result.add_engines(engine);
146
  })
147
  .or_insert_with(|| -> SearchResult { value });
148
  });
149
  }
150
  Err(error) => {
151
+ handle_error(&error, engine);
152
  }
153
  }
154
  }
155
 
156
+ if safe_search >= 3 {
157
+ let mut blacklist_map: HashMap<String, SearchResult> = HashMap::new();
158
+ filter_with_lists(
159
+ &mut result_map,
160
+ &mut blacklist_map,
161
+ file_path(FileType::BlockList)?,
162
+ )?;
163
 
164
+ filter_with_lists(
165
+ &mut blacklist_map,
166
+ &mut result_map,
167
+ file_path(FileType::AllowList)?,
168
+ )?;
169
 
170
+ drop(blacklist_map);
171
+ }
172
 
173
  let results: Vec<SearchResult> = result_map.into_values().collect();
174
 
175
+ Ok(SearchResults::new(results, query, &engine_errors_info))
 
 
 
 
176
  }
177
 
178
  /// Filters a map of search results using a list of regex patterns.
 
194
  let mut reader = BufReader::new(File::open(file_path)?);
195
 
196
  for line in reader.by_ref().lines() {
197
+ let re = Regex::new(line?.trim())?;
198
 
199
  // Iterate over each search result in the map and check if it matches the regex pattern
200
  for (url, search_result) in map_to_be_filtered.clone().into_iter() {
 
203
  || re.is_match(&search_result.description.to_lowercase())
204
  {
205
  // If the search result matches the regex pattern, move it from the original map to the resultant map
206
+ resultant_map.insert(
207
+ url.to_owned(),
208
+ map_to_be_filtered.remove(&url.to_owned()).unwrap(),
209
+ );
210
  }
211
  }
212
  }
 
217
  #[cfg(test)]
218
  mod tests {
219
  use super::*;
220
+ use smallvec::smallvec;
221
  use std::collections::HashMap;
222
  use std::io::Write;
223
  use tempfile::NamedTempFile;
 
227
  // Create a map of search results to filter
228
  let mut map_to_be_filtered = HashMap::new();
229
  map_to_be_filtered.insert(
230
+ "https://www.example.com".to_owned(),
231
  SearchResult {
232
+ title: "Example Domain".to_owned(),
233
+ url: "https://www.example.com".to_owned(),
234
  description: "This domain is for use in illustrative examples in documents."
235
+ .to_owned(),
236
+ engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
237
  },
238
  );
239
  map_to_be_filtered.insert(
240
+ "https://www.rust-lang.org/".to_owned(),
241
  SearchResult {
242
+ title: "Rust Programming Language".to_owned(),
243
+ url: "https://www.rust-lang.org/".to_owned(),
244
+ description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
245
+ engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
246
  },
247
  );
248
 
 
271
  fn test_filter_with_lists_wildcard() -> Result<(), Box<dyn std::error::Error>> {
272
  let mut map_to_be_filtered = HashMap::new();
273
  map_to_be_filtered.insert(
274
+ "https://www.example.com".to_owned(),
275
  SearchResult {
276
+ title: "Example Domain".to_owned(),
277
+ url: "https://www.example.com".to_owned(),
278
  description: "This domain is for use in illustrative examples in documents."
279
+ .to_owned(),
280
+ engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
281
  },
282
  );
283
  map_to_be_filtered.insert(
284
+ "https://www.rust-lang.org/".to_owned(),
285
  SearchResult {
286
+ title: "Rust Programming Language".to_owned(),
287
+ url: "https://www.rust-lang.org/".to_owned(),
288
+ description: "A systems programming language that runs blazingly fast, prevents segfaults, and guarantees thread safety.".to_owned(),
289
+ engine: smallvec!["Google".to_owned(), "DuckDuckGo".to_owned()],
290
  },
291
  );
292
 
 
331
  fn test_filter_with_lists_invalid_regex() {
332
  let mut map_to_be_filtered = HashMap::new();
333
  map_to_be_filtered.insert(
334
+ "https://www.example.com".to_owned(),
335
  SearchResult {
336
+ title: "Example Domain".to_owned(),
337
+ url: "https://www.example.com".to_owned(),
338
  description: "This domain is for use in illustrative examples in documents."
339
+ .to_owned(),
340
+ engine: smallvec!["Google".to_owned(), "Bing".to_owned()],
341
  },
342
  );
343
 
src/results/user_agent.rs CHANGED
@@ -1,30 +1,34 @@
1
  //! This module provides the functionality to generate random user agent string.
2
 
 
 
3
  use fake_useragent::{Browsers, UserAgents, UserAgentsBuilder};
4
 
5
  /// A static variable which stores the initially build `UserAgents` struct. So as it can be resused
6
  /// again and again without the need of reinitializing the `UserAgents` struct.
7
- static USER_AGENTS: once_cell::sync::Lazy<UserAgents> = once_cell::sync::Lazy::new(|| {
8
- UserAgentsBuilder::new()
9
- .cache(false)
10
- .dir("/tmp")
11
- .thread(1)
12
- .set_browsers(
13
- Browsers::new()
14
- .set_chrome()
15
- .set_safari()
16
- .set_edge()
17
- .set_firefox()
18
- .set_mozilla(),
19
- )
20
- .build()
21
- });
22
 
23
  /// A function to generate random user agent to improve privacy of the user.
24
  ///
25
  /// # Returns
26
  ///
27
  /// A randomly generated user agent string.
28
- pub fn random_user_agent() -> String {
29
- USER_AGENTS.random().to_string()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
30
  }
 
1
  //! This module provides the functionality to generate random user agent string.
2
 
3
+ use std::sync::OnceLock;
4
+
5
  use fake_useragent::{Browsers, UserAgents, UserAgentsBuilder};
6
 
7
  /// A static variable which stores the initially build `UserAgents` struct. So as it can be resused
8
  /// again and again without the need of reinitializing the `UserAgents` struct.
9
+ static USER_AGENTS: OnceLock<UserAgents> = OnceLock::new();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
10
 
11
  /// A function to generate random user agent to improve privacy of the user.
12
  ///
13
  /// # Returns
14
  ///
15
  /// A randomly generated user agent string.
16
+ pub fn random_user_agent() -> &'static str {
17
+ USER_AGENTS
18
+ .get_or_init(|| {
19
+ UserAgentsBuilder::new()
20
+ .cache(false)
21
+ .dir("/tmp")
22
+ .thread(1)
23
+ .set_browsers(
24
+ Browsers::new()
25
+ .set_chrome()
26
+ .set_safari()
27
+ .set_edge()
28
+ .set_firefox()
29
+ .set_mozilla(),
30
+ )
31
+ .build()
32
+ })
33
+ .random()
34
  }
src/server/routes/search.rs CHANGED
@@ -3,17 +3,73 @@
3
  use crate::{
4
  cache::cacher::RedisCache,
5
  config::parser::Config,
6
- models::{
7
- aggregation_models::SearchResults,
8
- engine_models::EngineHandler,
9
- server_models::{Cookie, SearchParams},
10
- },
11
  results::aggregator::aggregate,
12
  };
13
  use actix_web::{get, web, HttpRequest, HttpResponse};
14
  use handlebars::Handlebars;
 
 
 
 
 
 
15
  use tokio::join;
16
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  /// Handles the route of search page of the `websurfx` meta search engine website and it takes
18
  /// two search url parameters `q` and `page` where `page` parameter is optional.
19
  ///
@@ -47,42 +103,58 @@ pub async fn search(
47
  None => 1,
48
  };
49
 
 
 
 
 
 
 
 
 
 
 
 
50
  let (_, results, _) = join!(
51
  results(
52
  format!(
53
- "http://{}:{}/search?q={}&page={}",
54
  config.binding_ip,
55
  config.port,
56
  query,
57
- page - 1
 
58
  ),
59
  &config,
60
- query.to_string(),
61
  page - 1,
62
  req.clone(),
 
63
  ),
64
  results(
65
  format!(
66
- "http://{}:{}/search?q={}&page={}",
67
- config.binding_ip, config.port, query, page
68
  ),
69
  &config,
70
- query.to_string(),
71
  page,
72
  req.clone(),
 
73
  ),
74
  results(
75
  format!(
76
- "http://{}:{}/search?q={}&page={}",
77
  config.binding_ip,
78
  config.port,
79
  query,
80
- page + 1
 
81
  ),
82
  &config,
83
- query.to_string(),
84
  page + 1,
85
  req.clone(),
 
86
  )
87
  );
88
 
@@ -113,28 +185,54 @@ pub async fn search(
113
  async fn results(
114
  url: String,
115
  config: &Config,
116
- query: String,
117
  page: u32,
118
  req: HttpRequest,
 
119
  ) -> Result<SearchResults, Box<dyn std::error::Error>> {
120
  // Initialize redis cache connection struct
121
- let mut redis_cache = RedisCache::new(config.redis_url.clone())?;
 
 
 
 
 
 
122
  // fetch the cached results json.
123
- let cached_results_json = redis_cache.cached_json(&url);
 
124
  // check if fetched cache results was indeed fetched or it was an error and if so
125
  // handle the data accordingly.
126
  match cached_results_json {
127
- Ok(results) => Ok(serde_json::from_str::<SearchResults>(&results).unwrap()),
128
  Err(_) => {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
129
  // check if the cookie value is empty or not if it is empty then use the
130
  // default selected upstream search engines from the config file otherwise
131
  // parse the non-empty cookie and grab the user selected engines from the
132
  // UI and use that.
133
  let mut results: SearchResults = match req.cookie("appCookie") {
134
  Some(cookie_value) => {
135
- let cookie_value: Cookie = serde_json::from_str(cookie_value.name_value().1)?;
 
136
 
137
- let engines = cookie_value
138
  .engines
139
  .iter()
140
  .filter_map(|name| EngineHandler::new(name))
@@ -145,8 +243,9 @@ async fn results(
145
  page,
146
  config.aggregator.random_delay,
147
  config.debug,
148
- engines,
149
  config.request_timeout,
 
150
  )
151
  .await?
152
  }
@@ -156,15 +255,69 @@ async fn results(
156
  page,
157
  config.aggregator.random_delay,
158
  config.debug,
159
- config.upstream_search_engines.clone(),
160
  config.request_timeout,
 
161
  )
162
  .await?
163
  }
164
  };
165
- results.add_style(config.style.clone());
166
- redis_cache.cache_results(serde_json::to_string(&results)?, &url)?;
 
 
 
 
 
167
  Ok(results)
168
  }
169
  }
170
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  use crate::{
4
  cache::cacher::RedisCache,
5
  config::parser::Config,
6
+ handler::paths::{file_path, FileType},
7
+ models::{aggregation_models::SearchResults, engine_models::EngineHandler},
 
 
 
8
  results::aggregator::aggregate,
9
  };
10
  use actix_web::{get, web, HttpRequest, HttpResponse};
11
  use handlebars::Handlebars;
12
+ use regex::Regex;
13
+ use serde::Deserialize;
14
+ use std::{
15
+ fs::{read_to_string, File},
16
+ io::{BufRead, BufReader, Read},
17
+ };
18
  use tokio::join;
19
 
20
+ // ---- Constants ----
21
+ /// Initialize redis cache connection once and store it on the heap.
22
+ static REDIS_CACHE: async_once_cell::OnceCell<RedisCache> = async_once_cell::OnceCell::new();
23
+
24
+ /// A named struct which deserializes all the user provided search parameters and stores them.
25
+ #[derive(Deserialize)]
26
+ pub struct SearchParams {
27
+ /// It stores the search parameter option `q` (or query in simple words)
28
+ /// of the search url.
29
+ q: Option<String>,
30
+ /// It stores the search parameter `page` (or pageno in simple words)
31
+ /// of the search url.
32
+ page: Option<u32>,
33
+ /// It stores the search parameter `safesearch` (or safe search level in simple words) of the
34
+ /// search url.
35
+ safesearch: Option<u8>,
36
+ }
37
+
38
+ /// Handles the route of index page or main page of the `websurfx` meta search engine website.
39
+ #[get("/")]
40
+ pub async fn index(
41
+ hbs: web::Data<Handlebars<'_>>,
42
+ config: web::Data<Config>,
43
+ ) -> Result<HttpResponse, Box<dyn std::error::Error>> {
44
+ let page_content: String = hbs.render("index", &config.style).unwrap();
45
+ Ok(HttpResponse::Ok().body(page_content))
46
+ }
47
+
48
+ /// Handles the route of any other accessed route/page which is not provided by the
49
+ /// website essentially the 404 error page.
50
+ pub async fn not_found(
51
+ hbs: web::Data<Handlebars<'_>>,
52
+ config: web::Data<Config>,
53
+ ) -> Result<HttpResponse, Box<dyn std::error::Error>> {
54
+ let page_content: String = hbs.render("404", &config.style)?;
55
+
56
+ Ok(HttpResponse::Ok()
57
+ .content_type("text/html; charset=utf-8")
58
+ .body(page_content))
59
+ }
60
+
61
+ /// A named struct which is used to deserialize the cookies fetched from the client side.
62
+ #[allow(dead_code)]
63
+ #[derive(Deserialize)]
64
+ struct Cookie<'a> {
65
+ /// It stores the theme name used in the website.
66
+ theme: &'a str,
67
+ /// It stores the colorscheme name used for the website theme.
68
+ colorscheme: &'a str,
69
+ /// It stores the user selected upstream search engines selected from the UI.
70
+ engines: Vec<&'a str>,
71
+ }
72
+
73
  /// Handles the route of search page of the `websurfx` meta search engine website and it takes
74
  /// two search url parameters `q` and `page` where `page` parameter is optional.
75
  ///
 
103
  None => 1,
104
  };
105
 
106
+ let safe_search: u8 = match config.safe_search {
107
+ 3..=4 => config.safe_search,
108
+ _ => match &params.safesearch {
109
+ Some(safesearch) => match safesearch {
110
+ 0..=2 => *safesearch,
111
+ _ => 1,
112
+ },
113
+ None => config.safe_search,
114
+ },
115
+ };
116
+
117
  let (_, results, _) = join!(
118
  results(
119
  format!(
120
+ "http://{}:{}/search?q={}&page={}&safesearch={}",
121
  config.binding_ip,
122
  config.port,
123
  query,
124
+ page - 1,
125
+ safe_search
126
  ),
127
  &config,
128
+ query,
129
  page - 1,
130
  req.clone(),
131
+ safe_search
132
  ),
133
  results(
134
  format!(
135
+ "http://{}:{}/search?q={}&page={}&safesearch={}",
136
+ config.binding_ip, config.port, query, page, safe_search
137
  ),
138
  &config,
139
+ query,
140
  page,
141
  req.clone(),
142
+ safe_search
143
  ),
144
  results(
145
  format!(
146
+ "http://{}:{}/search?q={}&page={}&safesearch={}",
147
  config.binding_ip,
148
  config.port,
149
  query,
150
+ page + 1,
151
+ safe_search
152
  ),
153
  &config,
154
+ query,
155
  page + 1,
156
  req.clone(),
157
+ safe_search
158
  )
159
  );
160
 
 
185
  async fn results(
186
  url: String,
187
  config: &Config,
188
+ query: &str,
189
  page: u32,
190
  req: HttpRequest,
191
+ safe_search: u8,
192
  ) -> Result<SearchResults, Box<dyn std::error::Error>> {
193
  // Initialize redis cache connection struct
194
+ let mut redis_cache: RedisCache = REDIS_CACHE
195
+ .get_or_init(async {
196
+ // Initialize redis cache connection pool only one and store it in the heap.
197
+ RedisCache::new(&config.redis_url, 5).await.unwrap()
198
+ })
199
+ .await
200
+ .clone();
201
  // fetch the cached results json.
202
+ let cached_results_json: Result<String, error_stack::Report<crate::cache::error::PoolError>> =
203
+ redis_cache.clone().cached_json(&url).await;
204
  // check if fetched cache results was indeed fetched or it was an error and if so
205
  // handle the data accordingly.
206
  match cached_results_json {
207
+ Ok(results) => Ok(serde_json::from_str::<SearchResults>(&results)?),
208
  Err(_) => {
209
+ if safe_search == 4 {
210
+ let mut results: SearchResults = SearchResults::default();
211
+ let mut _flag: bool =
212
+ is_match_from_filter_list(file_path(FileType::BlockList)?, query)?;
213
+ _flag = !is_match_from_filter_list(file_path(FileType::AllowList)?, query)?;
214
+
215
+ if _flag {
216
+ results.set_disallowed();
217
+ results.add_style(&config.style);
218
+ results.set_page_query(query);
219
+ redis_cache
220
+ .cache_results(&serde_json::to_string(&results)?, &url)
221
+ .await?;
222
+ return Ok(results);
223
+ }
224
+ }
225
+
226
  // check if the cookie value is empty or not if it is empty then use the
227
  // default selected upstream search engines from the config file otherwise
228
  // parse the non-empty cookie and grab the user selected engines from the
229
  // UI and use that.
230
  let mut results: SearchResults = match req.cookie("appCookie") {
231
  Some(cookie_value) => {
232
+ let cookie_value: Cookie<'_> =
233
+ serde_json::from_str(cookie_value.name_value().1)?;
234
 
235
+ let engines: Vec<EngineHandler> = cookie_value
236
  .engines
237
  .iter()
238
  .filter_map(|name| EngineHandler::new(name))
 
243
  page,
244
  config.aggregator.random_delay,
245
  config.debug,
246
+ &engines,
247
  config.request_timeout,
248
+ safe_search,
249
  )
250
  .await?
251
  }
 
255
  page,
256
  config.aggregator.random_delay,
257
  config.debug,
258
+ &config.upstream_search_engines,
259
  config.request_timeout,
260
+ safe_search,
261
  )
262
  .await?
263
  }
264
  };
265
+ if results.engine_errors_info().is_empty() && results.results().is_empty() {
266
+ results.set_filtered();
267
+ }
268
+ results.add_style(&config.style);
269
+ redis_cache
270
+ .cache_results(&serde_json::to_string(&results)?, &url)
271
+ .await?;
272
  Ok(results)
273
  }
274
  }
275
  }
276
+
277
+ /// A helper function which checks whether the search query contains any keywords which should be
278
+ /// disallowed/allowed based on the regex based rules present in the blocklist and allowlist files.
279
+ fn is_match_from_filter_list(
280
+ file_path: &str,
281
+ query: &str,
282
+ ) -> Result<bool, Box<dyn std::error::Error>> {
283
+ let mut flag = false;
284
+ let mut reader = BufReader::new(File::open(file_path)?);
285
+ for line in reader.by_ref().lines() {
286
+ let re = Regex::new(&line?)?;
287
+ if re.is_match(query) {
288
+ flag = true;
289
+ break;
290
+ }
291
+ }
292
+ Ok(flag)
293
+ }
294
+
295
+ /// Handles the route of robots.txt page of the `websurfx` meta search engine website.
296
+ #[get("/robots.txt")]
297
+ pub async fn robots_data(_req: HttpRequest) -> Result<HttpResponse, Box<dyn std::error::Error>> {
298
+ let page_content: String =
299
+ read_to_string(format!("{}/robots.txt", file_path(FileType::Theme)?))?;
300
+ Ok(HttpResponse::Ok()
301
+ .content_type("text/plain; charset=ascii")
302
+ .body(page_content))
303
+ }
304
+
305
+ /// Handles the route of about page of the `websurfx` meta search engine website.
306
+ #[get("/about")]
307
+ pub async fn about(
308
+ hbs: web::Data<Handlebars<'_>>,
309
+ config: web::Data<Config>,
310
+ ) -> Result<HttpResponse, Box<dyn std::error::Error>> {
311
+ let page_content: String = hbs.render("about", &config.style)?;
312
+ Ok(HttpResponse::Ok().body(page_content))
313
+ }
314
+
315
+ /// Handles the route of settings page of the `websurfx` meta search engine website.
316
+ #[get("/settings")]
317
+ pub async fn settings(
318
+ hbs: web::Data<Handlebars<'_>>,
319
+ config: web::Data<Config>,
320
+ ) -> Result<HttpResponse, Box<dyn std::error::Error>> {
321
+ let page_content: String = hbs.render("settings", &config.style)?;
322
+ Ok(HttpResponse::Ok().body(page_content))
323
+ }
websurfx/config.lua CHANGED
@@ -10,6 +10,21 @@ production_use = false -- whether to use production mode or not (in other words
10
  -- if production_use is set to true
11
  -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
12
  request_timeout = 30 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds).
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
 
14
  -- ### Website ###
15
  -- The different colorschemes provided are:
@@ -34,4 +49,7 @@ theme = "simple" -- the theme name which should be used for the website
34
  redis_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
35
 
36
  -- ### Search Engines ###
37
- upstream_search_engines = { DuckDuckGo = true, Searx = false } -- select the upstream search engines from which the results should be fetched.
 
 
 
 
10
  -- if production_use is set to true
11
  -- There will be a random delay before sending the request to the search engines, this is to prevent DDoSing the upstream search engines from a large number of simultaneous requests.
12
  request_timeout = 30 -- timeout for the search requests sent to the upstream search engines to be fetched (value in seconds).
13
+ rate_limiter = {
14
+ number_of_requests = 20, -- The number of request that are allowed within a provided time limit.
15
+ time_limit = 3, -- The time limit in which the quantity of requests that should be accepted.
16
+ }
17
+
18
+ -- ### Search ###
19
+ -- Filter results based on different levels. The levels provided are:
20
+ -- {{
21
+ -- 0 - None
22
+ -- 1 - Low
23
+ -- 2 - Moderate
24
+ -- 3 - High
25
+ -- 4 - Aggressive
26
+ -- }}
27
+ safe_search = 2
28
 
29
  -- ### Website ###
30
  -- The different colorschemes provided are:
 
49
  redis_url = "redis://127.0.0.1:8082" -- redis connection url address on which the client should connect on.
50
 
51
  -- ### Search Engines ###
52
+ upstream_search_engines = {
53
+ DuckDuckGo = true,
54
+ Searx = false,
55
+ } -- select the upstream search engines from which the results should be fetched.