Evan Yang Spencer Najib alamin655 mergify[bot] neon_arch Spencerjibz spencer commited on
Commit
efa8efc
1 Parent(s): 33846ce

✨ Compression and encryption for the cached search results (#443)

Browse files

* attempt1

* rough draft

* add features and their optional dependancies

* add encryption and compression error variants

* add a sample implementation to cache trait

* Update src/cache/cacher.rs

Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>

* adjust comment so feature flag would apply?

* adjust feature flag so it applies?

* formatting

* Update src/cache/cacher.rs

update documentation

Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>

* [features]Add base64 and chacha20 dependencies for compress-cache-results and encrypt-cache-results

* move encryption key and cipher logic to separate sub module

* added cacha20 and cec-results feature

* added cacha20 and cec-results feature

* added compression and encryption helper functions to trait implementations

* added compression and encryption implementation for inMemoryCache

* base64 is only requried when redis-cache feature is enabled

* add error case for base64 and encryption/compression implementation to redisCache

* Refactor cacher to remove regex dependency

* fmt cache error and cacher

* Update Cargo.toml

disabling the unneeded default-features

Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>

* fix unused import warning for mimalloc

* remove deprecated method

* add doc comments for encryption module

* fix known bugs and use cfg-if module

* make cfg-if an optional dependency

* use feature-flag instead of maco lint

* add comment to explain type complexity

* bump app version

* Update src/cache/encryption.rs

Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>

* fixed type complexity and add docs for types

---------

Co-authored-by: Spencer Najib <spencernajib2@gmail.com>
Co-authored-by: alamin655 <mdalamin655@outlook.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: neon_arch <mustafadhuleb53@gmail.com>
Co-authored-by: Spencerjibz <=spencernajib2@gmail.com>
Co-authored-by: spencer <spencer@DESKTOP-SIF13AR>

Cargo.lock CHANGED
@@ -243,6 +243,16 @@ version = "1.0.2"
243
  source = "registry+https://github.com/rust-lang/crates.io-index"
244
  checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
245
 
 
 
 
 
 
 
 
 
 
 
246
  [[package]]
247
  name = "ahash"
248
  version = "0.7.7"
@@ -600,6 +610,30 @@ version = "1.0.0"
600
  source = "registry+https://github.com/rust-lang/crates.io-index"
601
  checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
602
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
603
  [[package]]
604
  name = "ci_info"
605
  version = "0.10.2"
@@ -636,6 +670,17 @@ dependencies = [
636
  "half",
637
  ]
638
 
 
 
 
 
 
 
 
 
 
 
 
639
  [[package]]
640
  name = "clap"
641
  version = "4.4.12"
@@ -897,6 +942,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
897
  checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
898
  dependencies = [
899
  "generic-array",
 
900
  "typenum",
901
  ]
902
 
@@ -1709,6 +1755,15 @@ dependencies = [
1709
  "hashbrown 0.14.3",
1710
  ]
1711
 
 
 
 
 
 
 
 
 
 
1712
  [[package]]
1713
  name = "iovec"
1714
  version = "0.1.4"
@@ -2219,6 +2274,12 @@ version = "11.1.3"
2219
  source = "registry+https://github.com/rust-lang/crates.io-index"
2220
  checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
2221
 
 
 
 
 
 
 
2222
  [[package]]
2223
  name = "openssl"
2224
  version = "0.10.62"
@@ -2521,6 +2582,17 @@ version = "0.3.28"
2521
  source = "registry+https://github.com/rust-lang/crates.io-index"
2522
  checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a"
2523
 
 
 
 
 
 
 
 
 
 
 
 
2524
  [[package]]
2525
  name = "powerfmt"
2526
  version = "0.2.0"
@@ -3392,6 +3464,12 @@ version = "0.3.0"
3392
  source = "registry+https://github.com/rust-lang/crates.io-index"
3393
  checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc"
3394
 
 
 
 
 
 
 
3395
  [[package]]
3396
  name = "syn"
3397
  version = "0.15.44"
@@ -3877,6 +3955,16 @@ version = "0.2.4"
3877
  source = "registry+https://github.com/rust-lang/crates.io-index"
3878
  checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
3879
 
 
 
 
 
 
 
 
 
 
 
3880
  [[package]]
3881
  name = "untrusted"
3882
  version = "0.9.0"
@@ -4058,7 +4146,7 @@ checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10"
4058
 
4059
  [[package]]
4060
  name = "websurfx"
4061
- version = "1.7.3"
4062
  dependencies = [
4063
  "actix-cors",
4064
  "actix-files",
@@ -4066,7 +4154,12 @@ dependencies = [
4066
  "actix-web",
4067
  "async-once-cell",
4068
  "async-trait",
 
4069
  "blake3",
 
 
 
 
4070
  "criterion",
4071
  "dhat",
4072
  "env_logger",
@@ -4328,3 +4421,9 @@ dependencies = [
4328
  "quote 1.0.33",
4329
  "syn 2.0.43",
4330
  ]
 
 
 
 
 
 
 
243
  source = "registry+https://github.com/rust-lang/crates.io-index"
244
  checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
245
 
246
+ [[package]]
247
+ name = "aead"
248
+ version = "0.5.2"
249
+ source = "registry+https://github.com/rust-lang/crates.io-index"
250
+ checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"
251
+ dependencies = [
252
+ "crypto-common",
253
+ "generic-array",
254
+ ]
255
+
256
  [[package]]
257
  name = "ahash"
258
  version = "0.7.7"
 
610
  source = "registry+https://github.com/rust-lang/crates.io-index"
611
  checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
612
 
613
+ [[package]]
614
+ name = "chacha20"
615
+ version = "0.9.1"
616
+ source = "registry+https://github.com/rust-lang/crates.io-index"
617
+ checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818"
618
+ dependencies = [
619
+ "cfg-if 1.0.0",
620
+ "cipher",
621
+ "cpufeatures",
622
+ ]
623
+
624
+ [[package]]
625
+ name = "chacha20poly1305"
626
+ version = "0.10.1"
627
+ source = "registry+https://github.com/rust-lang/crates.io-index"
628
+ checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35"
629
+ dependencies = [
630
+ "aead",
631
+ "chacha20",
632
+ "cipher",
633
+ "poly1305",
634
+ "zeroize",
635
+ ]
636
+
637
  [[package]]
638
  name = "ci_info"
639
  version = "0.10.2"
 
670
  "half",
671
  ]
672
 
673
+ [[package]]
674
+ name = "cipher"
675
+ version = "0.4.4"
676
+ source = "registry+https://github.com/rust-lang/crates.io-index"
677
+ checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
678
+ dependencies = [
679
+ "crypto-common",
680
+ "inout",
681
+ "zeroize",
682
+ ]
683
+
684
  [[package]]
685
  name = "clap"
686
  version = "4.4.12"
 
942
  checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
943
  dependencies = [
944
  "generic-array",
945
+ "rand_core 0.6.4",
946
  "typenum",
947
  ]
948
 
 
1755
  "hashbrown 0.14.3",
1756
  ]
1757
 
1758
+ [[package]]
1759
+ name = "inout"
1760
+ version = "0.1.3"
1761
+ source = "registry+https://github.com/rust-lang/crates.io-index"
1762
+ checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5"
1763
+ dependencies = [
1764
+ "generic-array",
1765
+ ]
1766
+
1767
  [[package]]
1768
  name = "iovec"
1769
  version = "0.1.4"
 
2274
  source = "registry+https://github.com/rust-lang/crates.io-index"
2275
  checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
2276
 
2277
+ [[package]]
2278
+ name = "opaque-debug"
2279
+ version = "0.3.0"
2280
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2281
+ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
2282
+
2283
  [[package]]
2284
  name = "openssl"
2285
  version = "0.10.62"
 
2582
  source = "registry+https://github.com/rust-lang/crates.io-index"
2583
  checksum = "69d3587f8a9e599cc7ec2c00e331f71c4e69a5f9a4b8a6efd5b07466b9736f9a"
2584
 
2585
+ [[package]]
2586
+ name = "poly1305"
2587
+ version = "0.8.0"
2588
+ source = "registry+https://github.com/rust-lang/crates.io-index"
2589
+ checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf"
2590
+ dependencies = [
2591
+ "cpufeatures",
2592
+ "opaque-debug",
2593
+ "universal-hash",
2594
+ ]
2595
+
2596
  [[package]]
2597
  name = "powerfmt"
2598
  version = "0.2.0"
 
3464
  source = "registry+https://github.com/rust-lang/crates.io-index"
3465
  checksum = "b1884d1bc09741d466d9b14e6d37ac89d6909cbcac41dd9ae982d4d063bbedfc"
3466
 
3467
+ [[package]]
3468
+ name = "subtle"
3469
+ version = "2.5.0"
3470
+ source = "registry+https://github.com/rust-lang/crates.io-index"
3471
+ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
3472
+
3473
  [[package]]
3474
  name = "syn"
3475
  version = "0.15.44"
 
3955
  source = "registry+https://github.com/rust-lang/crates.io-index"
3956
  checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
3957
 
3958
+ [[package]]
3959
+ name = "universal-hash"
3960
+ version = "0.5.1"
3961
+ source = "registry+https://github.com/rust-lang/crates.io-index"
3962
+ checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"
3963
+ dependencies = [
3964
+ "crypto-common",
3965
+ "subtle",
3966
+ ]
3967
+
3968
  [[package]]
3969
  name = "untrusted"
3970
  version = "0.9.0"
 
4146
 
4147
  [[package]]
4148
  name = "websurfx"
4149
+ version = "1.9.0"
4150
  dependencies = [
4151
  "actix-cors",
4152
  "actix-files",
 
4154
  "actix-web",
4155
  "async-once-cell",
4156
  "async-trait",
4157
+ "base64 0.21.5",
4158
  "blake3",
4159
+ "brotli",
4160
+ "cfg-if 1.0.0",
4161
+ "chacha20",
4162
+ "chacha20poly1305",
4163
  "criterion",
4164
  "dhat",
4165
  "env_logger",
 
4421
  "quote 1.0.33",
4422
  "syn 2.0.43",
4423
  ]
4424
+
4425
+ [[package]]
4426
+ name = "zeroize"
4427
+ version = "1.7.0"
4428
+ source = "registry+https://github.com/rust-lang/crates.io-index"
4429
+ checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d"
Cargo.toml CHANGED
@@ -1,6 +1,6 @@
1
  [package]
2
  name = "websurfx"
3
- version = "1.7.3"
4
  edition = "2021"
5
  description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
6
  repository = "https://github.com/neon-mmd/websurfx"
@@ -38,6 +38,11 @@ mimalloc = { version = "0.1.38", default-features = false }
38
  async-once-cell = {version="0.5.3", default-features=false}
39
  actix-governor = {version="0.5.0", default-features=false}
40
  mini-moka = { version="0.10", optional = true, default-features=false, features=["sync"]}
 
 
 
 
 
41
 
42
  [dev-dependencies]
43
  rusty-hook = {version="^0.11.2", default-features=false}
@@ -78,4 +83,8 @@ strip = "debuginfo"
78
  default = ["memory-cache"]
79
  dhat-heap = ["dep:dhat"]
80
  memory-cache = ["dep:mini-moka"]
81
- redis-cache = ["dep:redis"]
 
 
 
 
 
1
  [package]
2
  name = "websurfx"
3
+ version = "1.9.0"
4
  edition = "2021"
5
  description = "An open-source alternative to Searx that provides clean, ad-free, and organic results with incredible speed while keeping privacy and security in mind."
6
  repository = "https://github.com/neon-mmd/websurfx"
 
38
  async-once-cell = {version="0.5.3", default-features=false}
39
  actix-governor = {version="0.5.0", default-features=false}
40
  mini-moka = { version="0.10", optional = true, default-features=false, features=["sync"]}
41
+ brotli = { version = "3.4.0", default-features = false, features=["std"], optional=true}
42
+ chacha20poly1305={version="0.10.1", default-features=false, features=["alloc","getrandom"], optional=true}
43
+ chacha20 = {version="0.9.1", default-features=false, optional=true}
44
+ base64 = {version="0.21.5", default-features=false, features=["std"], optional=true}
45
+ cfg-if = {version="1.0.0", default-features=false,optional=true}
46
 
47
  [dev-dependencies]
48
  rusty-hook = {version="^0.11.2", default-features=false}
 
83
  default = ["memory-cache"]
84
  dhat-heap = ["dep:dhat"]
85
  memory-cache = ["dep:mini-moka"]
86
+ redis-cache = ["dep:redis","dep:base64"]
87
+ compress-cache-results = ["dep:brotli","dep:cfg-if"]
88
+ encrypt-cache-results = ["dep:chacha20poly1305","dep:chacha20"]
89
+ cec-cache-results = ["compress-cache-results","encrypt-cache-results"]
90
+
src/bin/websurfx.rs CHANGED
@@ -2,8 +2,9 @@
2
  //!
3
  //! This module contains the main function which handles the logging of the application to the
4
  //! stdout and handles the command line arguments provided and launches the `websurfx` server.
5
-
6
  use mimalloc::MiMalloc;
 
7
  use std::net::TcpListener;
8
  use websurfx::{cache::cacher::create_cache, config::parser::Config, run};
9
 
 
2
  //!
3
  //! This module contains the main function which handles the logging of the application to the
4
  //! stdout and handles the command line arguments provided and launches the `websurfx` server.
5
+ #[cfg(not(feature = "dhat-heap"))]
6
  use mimalloc::MiMalloc;
7
+
8
  use std::net::TcpListener;
9
  use websurfx::{cache::cacher::create_cache, config::parser::Config, run};
10
 
src/cache/cacher.rs CHANGED
@@ -4,6 +4,7 @@
4
  use error_stack::Report;
5
  #[cfg(feature = "memory-cache")]
6
  use mini_moka::sync::Cache as MokaCache;
 
7
  #[cfg(feature = "memory-cache")]
8
  use std::time::Duration;
9
  use tokio::sync::Mutex;
@@ -14,6 +15,9 @@ use super::error::CacheError;
14
  #[cfg(feature = "redis-cache")]
15
  use super::redis_cacher::RedisCache;
16
 
 
 
 
17
  /// Abstraction trait for common methods provided by a cache backend.
18
  #[async_trait::async_trait]
19
  pub trait Cacher: Send + Sync {
@@ -69,6 +73,237 @@ pub trait Cacher: Send + Sync {
69
  fn hash_url(&self, url: &str) -> String {
70
  blake3::hash(url.as_bytes()).to_string()
71
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
  }
73
 
74
  #[cfg(feature = "redis-cache")]
@@ -85,10 +320,14 @@ impl Cacher for RedisCache {
85
  }
86
 
87
  async fn cached_results(&mut self, url: &str) -> Result<SearchResults, Report<CacheError>> {
 
88
  let hashed_url_string: &str = &self.hash_url(url);
89
- let json = self.cached_json(hashed_url_string).await?;
90
- Ok(serde_json::from_str::<SearchResults>(&json)
91
- .map_err(|_| CacheError::SerializationError)?)
 
 
 
92
  }
93
 
94
  async fn cache_results(
@@ -96,10 +335,29 @@ impl Cacher for RedisCache {
96
  search_results: &SearchResults,
97
  url: &str,
98
  ) -> Result<(), Report<CacheError>> {
99
- let json =
100
- serde_json::to_string(search_results).map_err(|_| CacheError::SerializationError)?;
 
101
  let hashed_url_string = self.hash_url(url);
102
- self.cache_json(&json, &hashed_url_string).await
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
103
  }
104
  }
105
 
@@ -107,7 +365,7 @@ impl Cacher for RedisCache {
107
  #[cfg(feature = "memory-cache")]
108
  pub struct InMemoryCache {
109
  /// The backend cache which stores data.
110
- cache: MokaCache<String, SearchResults>,
111
  }
112
 
113
  #[cfg(feature = "memory-cache")]
@@ -126,7 +384,7 @@ impl Cacher for InMemoryCache {
126
  async fn cached_results(&mut self, url: &str) -> Result<SearchResults, Report<CacheError>> {
127
  let hashed_url_string = self.hash_url(url);
128
  match self.cache.get(&hashed_url_string) {
129
- Some(res) => Ok(res),
130
  None => Err(Report::new(CacheError::MissingValue)),
131
  }
132
  }
@@ -137,7 +395,8 @@ impl Cacher for InMemoryCache {
137
  url: &str,
138
  ) -> Result<(), Report<CacheError>> {
139
  let hashed_url_string = self.hash_url(url);
140
- self.cache.insert(hashed_url_string, search_results.clone());
 
141
  Ok(())
142
  }
143
  }
@@ -282,3 +541,5 @@ pub async fn create_cache(config: &Config) -> impl Cacher {
282
  #[cfg(not(any(feature = "memory-cache", feature = "redis-cache")))]
283
  return DisabledCache::build(config).await;
284
  }
 
 
 
4
  use error_stack::Report;
5
  #[cfg(feature = "memory-cache")]
6
  use mini_moka::sync::Cache as MokaCache;
7
+
8
  #[cfg(feature = "memory-cache")]
9
  use std::time::Duration;
10
  use tokio::sync::Mutex;
 
15
  #[cfg(feature = "redis-cache")]
16
  use super::redis_cacher::RedisCache;
17
 
18
+ #[cfg(any(feature = "encrypt-cache-results", feature = "cec-cache-results"))]
19
+ use super::encryption::*;
20
+
21
  /// Abstraction trait for common methods provided by a cache backend.
22
  #[async_trait::async_trait]
23
  pub trait Cacher: Send + Sync {
 
73
  fn hash_url(&self, url: &str) -> String {
74
  blake3::hash(url.as_bytes()).to_string()
75
  }
76
+
77
+ /// A helper function that returns either encrypted or decrypted results.
78
+ /// Feature flags (**encrypt-cache-results or cec-cache-results**) are required for this to work.
79
+ ///
80
+ /// # Arguments
81
+ ///
82
+ /// * `bytes` - It takes a slice of bytes as an argument.
83
+ /// * `encrypt` - A boolean to choose whether to encrypt or decrypt the bytes
84
+
85
+ ///
86
+ /// # Error
87
+ /// Returns either encrypted or decrypted bytes on success otherwise it returns a CacheError
88
+ /// on failure.
89
+ #[cfg(any(
90
+ // feature = "compress-cache-results",
91
+ feature = "encrypt-cache-results",
92
+ feature = "cec-cache-results"
93
+ ))]
94
+ fn encrypt_or_decrypt_results(
95
+ &mut self,
96
+ mut bytes: Vec<u8>,
97
+ encrypt: bool,
98
+ ) -> Result<Vec<u8>, Report<CacheError>> {
99
+ use chacha20poly1305::{
100
+ aead::{Aead, AeadCore, KeyInit, OsRng},
101
+ ChaCha20Poly1305,
102
+ };
103
+
104
+ let cipher = CIPHER.get_or_init(|| {
105
+ let key = ChaCha20Poly1305::generate_key(&mut OsRng);
106
+ ChaCha20Poly1305::new(&key)
107
+ });
108
+
109
+ let encryption_key = ENCRYPTION_KEY.get_or_init(
110
+ || ChaCha20Poly1305::generate_nonce(&mut OsRng), // 96-bits; unique per message
111
+ );
112
+
113
+ bytes = if encrypt {
114
+ cipher
115
+ .encrypt(encryption_key, bytes.as_ref())
116
+ .map_err(|_| CacheError::EncryptionError)?
117
+ } else {
118
+ cipher
119
+ .decrypt(encryption_key, bytes.as_ref())
120
+ .map_err(|_| CacheError::EncryptionError)?
121
+ };
122
+
123
+ Ok(bytes)
124
+ }
125
+
126
+ /// A helper function that returns compressed results.
127
+ /// Feature flags (**compress-cache-results or cec-cache-results**) are required for this to work.
128
+ ///
129
+ /// # Arguments
130
+ ///
131
+ /// * `bytes` - It takes a slice of bytes as an argument.
132
+
133
+ ///
134
+ /// # Error
135
+ /// Returns the compressed bytes on success otherwise it returns a CacheError
136
+ /// on failure.
137
+ #[cfg(any(feature = "compress-cache-results", feature = "cec-cache-results"))]
138
+ fn compress_results(&mut self, mut bytes: Vec<u8>) -> Result<Vec<u8>, Report<CacheError>> {
139
+ use std::io::Write;
140
+ let mut writer = brotli::CompressorWriter::new(Vec::new(), 4096, 11, 22);
141
+ writer
142
+ .write_all(&bytes)
143
+ .map_err(|_| CacheError::CompressionError)?;
144
+ bytes = writer.into_inner();
145
+ Ok(bytes)
146
+ }
147
+
148
+ /// A helper function that returns compressed-encrypted results.
149
+ /// Feature flag (**cec-cache-results**) is required for this to work.
150
+ ///
151
+ /// # Arguments
152
+ ///
153
+ /// * `bytes` - It takes a slice of bytes as an argument.
154
+
155
+ ///
156
+ /// # Error
157
+ /// Returns the compressed and encrypted bytes on success otherwise it returns a CacheError
158
+ /// on failure.
159
+ #[cfg(feature = "cec-cache-results")]
160
+ fn compress_encrypt_compress_results(
161
+ &mut self,
162
+ mut bytes: Vec<u8>,
163
+ ) -> Result<Vec<u8>, Report<CacheError>> {
164
+ // compress first
165
+ bytes = self.compress_results(bytes)?;
166
+ // encrypt
167
+ bytes = self.encrypt_or_decrypt_results(bytes, true)?;
168
+
169
+ // compress again;
170
+ bytes = self.compress_results(bytes)?;
171
+
172
+ Ok(bytes)
173
+ }
174
+
175
+ /// A helper function that returns compressed results.
176
+ /// Feature flags (**compress-cache-results or cec-cache-results**) are required for this to work.
177
+ /// If bytes where
178
+ /// # Arguments
179
+ ///
180
+ /// * `bytes` - It takes a slice of bytes as an argument.
181
+
182
+ ///
183
+ /// # Error
184
+ /// Returns the uncompressed bytes on success otherwise it returns a CacheError
185
+ /// on failure.
186
+
187
+ #[cfg(any(feature = "compress-cache-results", feature = "cec-cache-results"))]
188
+ fn decompress_results(&mut self, bytes: &[u8]) -> Result<Vec<u8>, Report<CacheError>> {
189
+ cfg_if::cfg_if! {
190
+ if #[cfg(feature = "compress-cache-results")]
191
+ {
192
+ decompress_util(bytes)
193
+
194
+ }
195
+ else if #[cfg(feature = "cec-cache-results")]
196
+ {
197
+ let decompressed = decompress_util(bytes)?;
198
+ let decrypted = self.encrypt_or_decrypt_results(decompressed, false)?;
199
+
200
+ decompress_util(&decrypted)
201
+
202
+ }
203
+ }
204
+ }
205
+
206
+ /// A helper function that compresses or encrypts search results before they're inserted into a cache store
207
+
208
+ /// # Arguments
209
+ ///
210
+ /// * `search_results` - A reference to the search_Results to process.
211
+ ///
212
+
213
+ ///
214
+ /// # Error
215
+ /// Returns a Vec of compressed or encrypted bytes on success otherwise it returns a CacheError
216
+ /// on failure.
217
+ fn pre_process_search_results(
218
+ &mut self,
219
+ search_results: &SearchResults,
220
+ ) -> Result<Vec<u8>, Report<CacheError>> {
221
+ #[allow(unused_mut)] // needs to be mutable when any of the features is enabled
222
+ let mut bytes: Vec<u8> = search_results.try_into()?;
223
+ #[cfg(feature = "compress-cache-results")]
224
+ {
225
+ let compressed = self.compress_results(bytes)?;
226
+ bytes = compressed;
227
+ }
228
+
229
+ #[cfg(feature = "encrypt-cache-results")]
230
+ {
231
+ let encrypted = self.encrypt_or_decrypt_results(bytes, true)?;
232
+ bytes = encrypted;
233
+ }
234
+
235
+ #[cfg(feature = "cec-cache-results")]
236
+ {
237
+ let compressed_encrypted_compressed = self.compress_encrypt_compress_results(bytes)?;
238
+ bytes = compressed_encrypted_compressed;
239
+ }
240
+
241
+ Ok(bytes)
242
+ }
243
+
244
+ /// A helper function that decompresses or decrypts search results after they're fetched from the cache-store
245
+
246
+ /// # Arguments
247
+ ///
248
+ /// * `bytes` - A Vec of bytes stores in the cache.
249
+ ///
250
+
251
+ ///
252
+ /// # Error
253
+ /// Returns the SearchResults struct on success otherwise it returns a CacheError
254
+ /// on failure.
255
+
256
+ #[allow(unused_mut)] // needs to be mutable when any of the features is enabled
257
+ fn post_process_search_results(
258
+ &mut self,
259
+ mut bytes: Vec<u8>,
260
+ ) -> Result<SearchResults, Report<CacheError>> {
261
+ #[cfg(feature = "compress-cache-results")]
262
+ {
263
+ let decompressed = self.decompress_results(&bytes)?;
264
+ bytes = decompressed
265
+ }
266
+
267
+ #[cfg(feature = "encrypt-cache-results")]
268
+ {
269
+ let decrypted = self.encrypt_or_decrypt_results(bytes, false)?;
270
+ bytes = decrypted
271
+ }
272
+
273
+ #[cfg(feature = "cec-cache-results")]
274
+ {
275
+ let decompressed_decrypted = self.decompress_results(&bytes)?;
276
+ bytes = decompressed_decrypted;
277
+ }
278
+
279
+ Ok(bytes.try_into()?)
280
+ }
281
+ }
282
+
283
+ /// A helper function that returns compressed results.
284
+ /// Feature flags (**compress-cache-results or cec-cache-results**) are required for this to work.
285
+ /// If bytes where
286
+ /// # Arguments
287
+ ///
288
+ /// * `bytes` - It takes a slice of bytes as an argument.
289
+
290
+ ///
291
+ /// # Error
292
+ /// Returns the uncompressed bytes on success otherwise it returns a CacheError
293
+ /// on failure.
294
+
295
+ #[cfg(any(feature = "compress-cache-results", feature = "cec-cache-results"))]
296
+ fn decompress_util(input: &[u8]) -> Result<Vec<u8>, Report<CacheError>> {
297
+ use std::io::Write;
298
+ let mut writer = brotli::DecompressorWriter::new(Vec::new(), 4096);
299
+
300
+ writer
301
+ .write_all(input)
302
+ .map_err(|_| CacheError::CompressionError)?;
303
+ let bytes = writer
304
+ .into_inner()
305
+ .map_err(|_| CacheError::CompressionError)?;
306
+ Ok(bytes)
307
  }
308
 
309
  #[cfg(feature = "redis-cache")]
 
320
  }
321
 
322
  async fn cached_results(&mut self, url: &str) -> Result<SearchResults, Report<CacheError>> {
323
+ use base64::Engine;
324
  let hashed_url_string: &str = &self.hash_url(url);
325
+ let base64_string = self.cached_json(hashed_url_string).await?;
326
+
327
+ let bytes = base64::engine::general_purpose::STANDARD_NO_PAD
328
+ .decode(base64_string)
329
+ .map_err(|_| CacheError::Base64DecodingOrEncodingError)?;
330
+ self.post_process_search_results(bytes)
331
  }
332
 
333
  async fn cache_results(
 
335
  search_results: &SearchResults,
336
  url: &str,
337
  ) -> Result<(), Report<CacheError>> {
338
+ use base64::Engine;
339
+ let bytes = self.pre_process_search_results(search_results)?;
340
+ let base64_string = base64::engine::general_purpose::STANDARD_NO_PAD.encode(bytes);
341
  let hashed_url_string = self.hash_url(url);
342
+ self.cache_json(&base64_string, &hashed_url_string).await
343
+ }
344
+ }
345
+ /// TryInto implementation for SearchResults from Vec<u8>
346
+ use std::convert::TryInto;
347
+
348
+ impl TryInto<SearchResults> for Vec<u8> {
349
+ type Error = CacheError;
350
+
351
+ fn try_into(self) -> Result<SearchResults, Self::Error> {
352
+ serde_json::from_slice(&self).map_err(|_| CacheError::SerializationError)
353
+ }
354
+ }
355
+
356
+ impl TryInto<Vec<u8>> for &SearchResults {
357
+ type Error = CacheError;
358
+
359
+ fn try_into(self) -> Result<Vec<u8>, Self::Error> {
360
+ serde_json::to_vec(self).map_err(|_| CacheError::SerializationError)
361
  }
362
  }
363
 
 
365
  #[cfg(feature = "memory-cache")]
366
  pub struct InMemoryCache {
367
  /// The backend cache which stores data.
368
+ cache: MokaCache<String, Vec<u8>>,
369
  }
370
 
371
  #[cfg(feature = "memory-cache")]
 
384
  async fn cached_results(&mut self, url: &str) -> Result<SearchResults, Report<CacheError>> {
385
  let hashed_url_string = self.hash_url(url);
386
  match self.cache.get(&hashed_url_string) {
387
+ Some(res) => self.post_process_search_results(res),
388
  None => Err(Report::new(CacheError::MissingValue)),
389
  }
390
  }
 
395
  url: &str,
396
  ) -> Result<(), Report<CacheError>> {
397
  let hashed_url_string = self.hash_url(url);
398
+ let bytes = self.pre_process_search_results(search_results)?;
399
+ self.cache.insert(hashed_url_string, bytes);
400
  Ok(())
401
  }
402
  }
 
541
  #[cfg(not(any(feature = "memory-cache", feature = "redis-cache")))]
542
  return DisabledCache::build(config).await;
543
  }
544
+
545
+ //#[cfg(feature = "Compress-cache-results")]
src/cache/encryption.rs ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use chacha20poly1305::{
2
+ consts::{B0, B1},
3
+ ChaChaPoly1305,
4
+ };
5
+ use std::sync::OnceLock;
6
+
7
+ use chacha20::{
8
+ cipher::{
9
+ generic_array::GenericArray,
10
+ typenum::{UInt, UTerm},
11
+ StreamCipherCoreWrapper,
12
+ },
13
+ ChaChaCore,
14
+ };
15
+
16
+ /// The ChaCha20 core wrapped in a stream cipher for use in ChaCha20-Poly1305 authenticated encryption.
17
+ type StreamCipherCoreWrapperType =
18
+ StreamCipherCoreWrapper<ChaChaCore<UInt<UInt<UInt<UInt<UTerm, B1>, B0>, B1>, B0>>>;
19
+ /// Our ChaCha20-Poly1305 cipher instance, lazily initialized.
20
+ pub static CIPHER: OnceLock<ChaChaPoly1305<StreamCipherCoreWrapperType>> = OnceLock::new();
21
+
22
+ /// The type alias for our encryption key, a 32-byte array.
23
+ type GenericArrayType = GenericArray<u8, UInt<UInt<UInt<UInt<UTerm, B1>, B1>, B0>, B0>>;
24
+ /// Our encryption key, lazily initialized.
25
+ pub static ENCRYPTION_KEY: OnceLock<GenericArrayType> = OnceLock::new();
src/cache/error.rs CHANGED
@@ -18,6 +18,12 @@ pub enum CacheError {
18
  SerializationError,
19
  /// Returned when the value is missing.
20
  MissingValue,
 
 
 
 
 
 
21
  }
22
 
23
  impl fmt::Display for CacheError {
@@ -43,6 +49,18 @@ impl fmt::Display for CacheError {
43
  CacheError::SerializationError => {
44
  write!(f, "Unable to serialize, deserialize from the cache")
45
  }
 
 
 
 
 
 
 
 
 
 
 
 
46
  }
47
  }
48
  }
 
18
  SerializationError,
19
  /// Returned when the value is missing.
20
  MissingValue,
21
+ /// whenever encryption or decryption of cache results fails
22
+ EncryptionError,
23
+ /// Whenever compression of the cache results fails
24
+ CompressionError,
25
+ /// Whenever base64 decoding failed
26
+ Base64DecodingOrEncodingError,
27
  }
28
 
29
  impl fmt::Display for CacheError {
 
49
  CacheError::SerializationError => {
50
  write!(f, "Unable to serialize, deserialize from the cache")
51
  }
52
+
53
+ CacheError::EncryptionError => {
54
+ write!(f, "Failed to encrypt or decrypt cache-results")
55
+ }
56
+
57
+ CacheError::CompressionError => {
58
+ write!(f, "failed to compress or uncompress cache results")
59
+ }
60
+
61
+ CacheError::Base64DecodingOrEncodingError => {
62
+ write!(f, "base64 encoding or decoding failed")
63
+ }
64
  }
65
  }
66
  }
src/cache/mod.rs CHANGED
@@ -1,7 +1,11 @@
1
  //! This module provides the modules which provide the functionality to cache the aggregated
2
  //! results fetched and aggregated from the upstream search engines in a json format.
3
-
4
  pub mod cacher;
 
 
 
 
5
  pub mod error;
 
6
  #[cfg(feature = "redis-cache")]
7
  pub mod redis_cacher;
 
1
  //! This module provides the modules which provide the functionality to cache the aggregated
2
  //! results fetched and aggregated from the upstream search engines in a json format.
 
3
  pub mod cacher;
4
+
5
+ #[cfg(any(feature = "encrypt-cache-results", feature = "cec-cache-results"))]
6
+ /// encryption module contains encryption utils such the cipher and key
7
+ pub mod encryption;
8
  pub mod error;
9
+
10
  #[cfg(feature = "redis-cache")]
11
  pub mod redis_cacher;
src/cache/redis_cacher.rs CHANGED
@@ -44,7 +44,7 @@ impl RedisCache {
44
  let mut tasks: Vec<_> = Vec::new();
45
 
46
  for _ in 0..pool_size {
47
- tasks.push(client.get_tokio_connection_manager());
48
  }
49
 
50
  let redis_cache = RedisCache {
 
44
  let mut tasks: Vec<_> = Vec::new();
45
 
46
  for _ in 0..pool_size {
47
+ tasks.push(client.get_connection_manager());
48
  }
49
 
50
  let redis_cache = RedisCache {