| 1 | // Copyright 2022 The Matrix.org Foundation C.I.C. |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | #![cfg_attr( |
| 16 | not(any(feature = "state-store", feature = "crypto-store", feature = "event-cache")), |
| 17 | allow(dead_code, unused_imports) |
| 18 | )] |
| 19 | |
| 20 | mod connection; |
| 21 | #[cfg(feature = "crypto-store")] |
| 22 | mod crypto_store; |
| 23 | mod error; |
| 24 | #[cfg(feature = "event-cache")] |
| 25 | mod event_cache_store; |
| 26 | #[cfg(feature = "event-cache")] |
| 27 | mod media_store; |
| 28 | #[cfg(feature = "state-store")] |
| 29 | mod state_store; |
| 30 | mod utils; |
| 31 | use std::{ |
| 32 | cmp::max, |
| 33 | fmt, |
| 34 | path::{Path, PathBuf}, |
| 35 | }; |
| 36 | |
| 37 | use deadpool::managed::PoolConfig; |
| 38 | use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; |
| 39 | |
| 40 | #[cfg(feature = "crypto-store")] |
| 41 | pub use self::crypto_store::SqliteCryptoStore; |
| 42 | pub use self::error::OpenStoreError; |
| 43 | #[cfg(feature = "event-cache")] |
| 44 | pub use self::event_cache_store::SqliteEventCacheStore; |
| 45 | #[cfg(feature = "event-cache")] |
| 46 | pub use self::media_store::SqliteMediaStore; |
| 47 | #[cfg(feature = "state-store")] |
| 48 | pub use self::state_store::{DATABASE_NAME as STATE_STORE_DATABASE_NAME, SqliteStateStore}; |
| 49 | |
| 50 | #[cfg(test)] |
| 51 | matrix_sdk_test_utils::init_tracing_for_tests!(); |
| 52 | |
| 53 | /// An enum used to store the secret that gives access to a store |
| 54 | #[derive(Clone, Debug, PartialEq, Zeroize, ZeroizeOnDrop)] |
| 55 | pub enum Secret { |
| 56 | // Cryptographic key used to open the store |
| 57 | Key(Box<[u8; 32]>), |
| 58 | // Passphrase used to open the store |
| 59 | PassPhrase(Zeroizing<String>), |
| 60 | } |
| 61 | |
| 62 | /// A configuration structure used for opening a store. |
| 63 | #[derive(Clone)] |
| 64 | pub struct SqliteStoreConfig { |
| 65 | /// Path to the database, without the file name. |
| 66 | path: PathBuf, |
| 67 | /// Secret to open the store, if any |
| 68 | secret: Option<Secret>, |
| 69 | /// The pool configuration for [`deadpool`]. |
| 70 | pool_config: PoolConfig, |
| 71 | /// The runtime configuration to apply when opening an SQLite connection. |
| 72 | runtime_config: RuntimeConfig, |
| 73 | } |
| 74 | |
| 75 | impl fmt::Debug for SqliteStoreConfig { |
| 76 | fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { |
| 77 | formatter |
| 78 | .debug_struct("SqliteStoreConfig") |
| 79 | .field("path", &self.path) |
| 80 | .field("pool_config", &self.pool_config) |
| 81 | .field("runtime_config", &self.runtime_config) |
| 82 | .finish_non_exhaustive() |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | /// The minimum size of the connections pool. |
| 87 | /// |
| 88 | /// We need at least 2 connections: one connection for write operations, and one |
| 89 | /// connection for read operations. |
| 90 | const POOL_MINIMUM_SIZE: usize = 2; |
| 91 | |
| 92 | impl SqliteStoreConfig { |
| 93 | /// Create a new [`SqliteStoreConfig`] with a path representing the |
| 94 | /// directory containing the store database. |
| 95 | pub fn new<P>(path: P) -> Self |
| 96 | where |
| 97 | P: AsRef<Path>, |
| 98 | { |
| 99 | Self { |
| 100 | path: path.as_ref().to_path_buf(), |
| 101 | pool_config: PoolConfig::new(max(POOL_MINIMUM_SIZE, num_cpus::get_physical() * 4)), |
| 102 | runtime_config: RuntimeConfig::default(), |
| 103 | secret: None, |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | /// Similar to [`SqliteStoreConfig::new`], but with defaults tailored for a |
| 108 | /// low memory usage environment. |
| 109 | /// |
| 110 | /// The following defaults are set: |
| 111 | /// |
| 112 | /// * The `pool_max_size` is set to the number of physical CPU, so one |
| 113 | /// connection per physical thread, |
| 114 | /// * The `cache_size` is set to 500Kib, |
| 115 | /// * The `journal_size_limit` is set to 2Mib. |
| 116 | pub fn with_low_memory_config<P>(path: P) -> Self |
| 117 | where |
| 118 | P: AsRef<Path>, |
| 119 | { |
| 120 | Self::new(path) |
| 121 | // Maximum one connection per physical thread. |
| 122 | .pool_max_size(num_cpus::get_physical()) |
| 123 | // Cache size is 500Kib. |
| 124 | .cache_size(500_000) |
| 125 | // Journal size limit is 2Mib. |
| 126 | .journal_size_limit(2_000_000) |
| 127 | } |
| 128 | |
| 129 | /// Override the path. |
| 130 | pub fn path<P>(mut self, path: P) -> Self |
| 131 | where |
| 132 | P: AsRef<Path>, |
| 133 | { |
| 134 | self.path = path.as_ref().to_path_buf(); |
| 135 | self |
| 136 | } |
| 137 | |
| 138 | /// Define the passphrase if the store is encoded. |
| 139 | pub fn passphrase(mut self, passphrase: Option<&str>) -> Self { |
| 140 | self.secret = |
| 141 | passphrase.map(|passphrase| Secret::PassPhrase(Zeroizing::new(passphrase.to_owned()))); |
| 142 | self |
| 143 | } |
| 144 | |
| 145 | /// Define the key if the store is encoded. |
| 146 | pub fn key(mut self, key: Option<&[u8; 32]>) -> Self { |
| 147 | self.secret = key.map(|key| Secret::Key(Box::new(*key))); |
| 148 | self |
| 149 | } |
| 150 | |
| 151 | /// Define the maximum pool size for [`deadpool`]. |
| 152 | /// |
| 153 | /// See [`deadpool::managed::PoolConfig::max_size`] to learn more. |
| 154 | pub fn pool_max_size(mut self, max_size: usize) -> Self { |
| 155 | self.pool_config.max_size = max(POOL_MINIMUM_SIZE, max_size); |
| 156 | self |
| 157 | } |
| 158 | |
| 159 | /// Optimize the database. |
| 160 | /// |
| 161 | /// The SQLite documentation recommends to run this regularly and after any |
| 162 | /// schema change. The easiest is to do it consistently when the store is |
| 163 | /// constructed, after eventual migrations. |
| 164 | /// |
| 165 | /// See [`PRAGMA optimize`] to learn more. |
| 166 | /// |
| 167 | /// The default value is `true`. |
| 168 | /// |
| 169 | /// [`PRAGMA optimize`]: https://www.sqlite.org/pragma.html#pragma_optimize |
| 170 | pub fn optimize(mut self, optimize: bool) -> Self { |
| 171 | self.runtime_config.optimize = optimize; |
| 172 | self |
| 173 | } |
| 174 | |
| 175 | /// Define the maximum size in **bytes** the SQLite cache can use. |
| 176 | /// |
| 177 | /// See [`PRAGMA cache_size`] to learn more. |
| 178 | /// |
| 179 | /// The default value is 2Mib. |
| 180 | /// |
| 181 | /// [`PRAGMA cache_size`]: https://www.sqlite.org/pragma.html#pragma_cache_size |
| 182 | pub fn cache_size(mut self, cache_size: u32) -> Self { |
| 183 | self.runtime_config.cache_size = cache_size; |
| 184 | self |
| 185 | } |
| 186 | |
| 187 | /// Limit the size of the WAL file, in **bytes**. |
| 188 | /// |
| 189 | /// By default, while the DB connections of the databases are open, [the |
| 190 | /// size of the WAL file can keep increasing][size_wal_file] depending on |
| 191 | /// the size needed for the transactions. A critical case is `VACUUM` |
| 192 | /// which basically writes the content of the DB file to the WAL file |
| 193 | /// before writing it back to the DB file, so we end up taking twice the |
| 194 | /// size of the database. |
| 195 | /// |
| 196 | /// By setting this limit, the WAL file is truncated after its content is |
| 197 | /// written to the database, if it is bigger than the limit. |
| 198 | /// |
| 199 | /// See [`PRAGMA journal_size_limit`] to learn more. The value `limit` |
| 200 | /// corresponds to `N` in `PRAGMA journal_size_limit = N`. |
| 201 | /// |
| 202 | /// The default value is 10Mib. |
| 203 | /// |
| 204 | /// [size_wal_file]: https://www.sqlite.org/wal.html#avoiding_excessively_large_wal_files |
| 205 | /// [`PRAGMA journal_size_limit`]: https://www.sqlite.org/pragma.html#pragma_journal_size_limit |
| 206 | pub fn journal_size_limit(mut self, limit: u32) -> Self { |
| 207 | self.runtime_config.journal_size_limit = limit; |
| 208 | self |
| 209 | } |
| 210 | |
| 211 | /// Returns the pool configuration. |
| 212 | pub(crate) fn pool_config(&self) -> PoolConfig { |
| 213 | self.pool_config |
| 214 | } |
| 215 | |
| 216 | /// Returns the runtime configuration. |
| 217 | pub(crate) fn runtime_config(&self) -> RuntimeConfig { |
| 218 | self.runtime_config |
| 219 | } |
| 220 | |
| 221 | /// Build a pool of active connections to a particular database. |
| 222 | pub fn build_pool_of_connections( |
| 223 | &self, |
| 224 | database_name: &str, |
| 225 | ) -> Result<connection::Pool, connection::CreatePoolError> { |
| 226 | let path = self.path.join(database_name); |
| 227 | let manager = connection::Manager::new(path); |
| 228 | |
| 229 | connection::Pool::builder(manager) |
| 230 | .config(self.pool_config) |
| 231 | .runtime(connection::RUNTIME) |
| 232 | .build() |
| 233 | .map_err(connection::CreatePoolError::Build) |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | /// This type represents values to set at runtime when a database is opened. |
| 238 | /// |
| 239 | /// This configuration is applied by |
| 240 | /// [`utils::SqliteAsyncConnExt::apply_runtime_config`]. |
| 241 | #[derive(Clone, Copy, Debug)] |
| 242 | struct RuntimeConfig { |
| 243 | /// If `true`, [`utils::SqliteAsyncConnExt::optimize`] will be called. |
| 244 | optimize: bool, |
| 245 | |
| 246 | /// Regardless of the value, [`utils::SqliteAsyncConnExt::cache_size`] will |
| 247 | /// always be called with this value. |
| 248 | cache_size: u32, |
| 249 | |
| 250 | /// Regardless of the value, |
| 251 | /// [`utils::SqliteAsyncConnExt::journal_size_limit`] will always be called |
| 252 | /// with this value. |
| 253 | journal_size_limit: u32, |
| 254 | } |
| 255 | |
| 256 | impl Default for RuntimeConfig { |
| 257 | fn default() -> Self { |
| 258 | Self { |
| 259 | // Optimize is always applied. |
| 260 | optimize: true, |
| 261 | // A cache of 2Mib. |
| 262 | cache_size: 2_000_000, |
| 263 | // A limit of 10Mib. |
| 264 | journal_size_limit: 10_000_000, |
| 265 | } |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | #[cfg(test)] |
| 270 | mod tests { |
| 271 | use std::{ |
| 272 | ops::Not, |
| 273 | path::{Path, PathBuf}, |
| 274 | }; |
| 275 | |
| 276 | use super::{POOL_MINIMUM_SIZE, Secret, SqliteStoreConfig}; |
| 277 | |
| 278 | #[test] |
| 279 | fn test_new() { |
| 280 | let store_config = SqliteStoreConfig::new(Path::new("foo")); |
| 281 | |
| 282 | assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical() * 4); |
| 283 | assert!(store_config.runtime_config.optimize); |
| 284 | assert_eq!(store_config.runtime_config.cache_size, 2_000_000); |
| 285 | assert_eq!(store_config.runtime_config.journal_size_limit, 10_000_000); |
| 286 | } |
| 287 | |
| 288 | #[test] |
| 289 | fn test_with_low_memory_config() { |
| 290 | let store_config = SqliteStoreConfig::with_low_memory_config(Path::new("foo")); |
| 291 | |
| 292 | assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical()); |
| 293 | assert!(store_config.runtime_config.optimize); |
| 294 | assert_eq!(store_config.runtime_config.cache_size, 500_000); |
| 295 | assert_eq!(store_config.runtime_config.journal_size_limit, 2_000_000); |
| 296 | } |
| 297 | |
| 298 | #[test] |
| 299 | fn test_store_config_when_passphrase() { |
| 300 | let store_config = SqliteStoreConfig::new(Path::new("foo")) |
| 301 | .passphrase(Some("bar")) |
| 302 | .pool_max_size(42) |
| 303 | .optimize(false) |
| 304 | .cache_size(43) |
| 305 | .journal_size_limit(44); |
| 306 | |
| 307 | assert_eq!(store_config.path, PathBuf::from("foo")); |
| 308 | assert_eq!(store_config.secret, Some(Secret::PassPhrase("bar".to_owned().into()))); |
| 309 | assert_eq!(store_config.pool_config.max_size, 42); |
| 310 | assert!(store_config.runtime_config.optimize.not()); |
| 311 | assert_eq!(store_config.runtime_config.cache_size, 43); |
| 312 | assert_eq!(store_config.runtime_config.journal_size_limit, 44); |
| 313 | } |
| 314 | |
| 315 | #[test] |
| 316 | fn test_store_config_when_key() { |
| 317 | let store_config = SqliteStoreConfig::new(Path::new("foo")) |
| 318 | .key(Some(&[ |
| 319 | 143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61, |
| 320 | 107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156, |
| 321 | ])) |
| 322 | .pool_max_size(42) |
| 323 | .optimize(false) |
| 324 | .cache_size(43) |
| 325 | .journal_size_limit(44); |
| 326 | |
| 327 | assert_eq!(store_config.path, PathBuf::from("foo")); |
| 328 | assert_eq!( |
| 329 | store_config.secret, |
| 330 | Some(Secret::Key(Box::new([ |
| 331 | 143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61, |
| 332 | 107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156, |
| 333 | ]))) |
| 334 | ); |
| 335 | assert_eq!(store_config.pool_config.max_size, 42); |
| 336 | assert!(store_config.runtime_config.optimize.not()); |
| 337 | assert_eq!(store_config.runtime_config.cache_size, 43); |
| 338 | assert_eq!(store_config.runtime_config.journal_size_limit, 44); |
| 339 | } |
| 340 | |
| 341 | #[test] |
| 342 | fn test_store_config_path() { |
| 343 | let store_config = SqliteStoreConfig::new(Path::new("foo")).path(Path::new("bar")); |
| 344 | |
| 345 | assert_eq!(store_config.path, PathBuf::from("bar")); |
| 346 | } |
| 347 | |
| 348 | #[test] |
| 349 | fn test_pool_size_has_a_minimum() { |
| 350 | let store_config = SqliteStoreConfig::new(Path::new("foo")).pool_max_size(1); |
| 351 | |
| 352 | assert_eq!(store_config.pool_config.max_size, POOL_MINIMUM_SIZE); |
| 353 | } |
| 354 | } |