atagen

matrix-rust-sdk

public
Star0
Watch0
ssh://brewery@pls.lobotomise.me:33/atagen/matrix-rust-sdk.git
crates/matrix-sdk-sqlite/src/lib.rsRust354 lines11.9 KB
1
// Copyright 2022 The Matrix.org Foundation C.I.C.
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14
15
#![cfg_attr(
16
    not(any(feature = "state-store", feature = "crypto-store", feature = "event-cache")),
17
    allow(dead_code, unused_imports)
18
)]
19
20
mod connection;
21
#[cfg(feature = "crypto-store")]
22
mod crypto_store;
23
mod error;
24
#[cfg(feature = "event-cache")]
25
mod event_cache_store;
26
#[cfg(feature = "event-cache")]
27
mod media_store;
28
#[cfg(feature = "state-store")]
29
mod state_store;
30
mod utils;
31
use std::{
32
    cmp::max,
33
    fmt,
34
    path::{Path, PathBuf},
35
};
36
37
use deadpool::managed::PoolConfig;
38
use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing};
39
40
#[cfg(feature = "crypto-store")]
41
pub use self::crypto_store::SqliteCryptoStore;
42
pub use self::error::OpenStoreError;
43
#[cfg(feature = "event-cache")]
44
pub use self::event_cache_store::SqliteEventCacheStore;
45
#[cfg(feature = "event-cache")]
46
pub use self::media_store::SqliteMediaStore;
47
#[cfg(feature = "state-store")]
48
pub use self::state_store::{DATABASE_NAME as STATE_STORE_DATABASE_NAME, SqliteStateStore};
49
50
#[cfg(test)]
51
matrix_sdk_test_utils::init_tracing_for_tests!();
52
53
/// An enum used to store the secret that gives access to a store
54
#[derive(Clone, Debug, PartialEq, Zeroize, ZeroizeOnDrop)]
55
pub enum Secret {
56
    // Cryptographic key used to open the store
57
    Key(Box<[u8; 32]>),
58
    // Passphrase used to open the store
59
    PassPhrase(Zeroizing<String>),
60
}
61
62
/// A configuration structure used for opening a store.
63
#[derive(Clone)]
64
pub struct SqliteStoreConfig {
65
    /// Path to the database, without the file name.
66
    path: PathBuf,
67
    /// Secret to open the store, if any
68
    secret: Option<Secret>,
69
    /// The pool configuration for [`deadpool`].
70
    pool_config: PoolConfig,
71
    /// The runtime configuration to apply when opening an SQLite connection.
72
    runtime_config: RuntimeConfig,
73
}
74
75
impl fmt::Debug for SqliteStoreConfig {
76
    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
77
        formatter
78
            .debug_struct("SqliteStoreConfig")
79
            .field("path", &self.path)
80
            .field("pool_config", &self.pool_config)
81
            .field("runtime_config", &self.runtime_config)
82
            .finish_non_exhaustive()
83
    }
84
}
85
86
/// The minimum size of the connections pool.
87
///
88
/// We need at least 2 connections: one connection for write operations, and one
89
/// connection for read operations.
90
const POOL_MINIMUM_SIZE: usize = 2;
91
92
impl SqliteStoreConfig {
93
    /// Create a new [`SqliteStoreConfig`] with a path representing the
94
    /// directory containing the store database.
95
    pub fn new<P>(path: P) -> Self
96
    where
97
        P: AsRef<Path>,
98
    {
99
        Self {
100
            path: path.as_ref().to_path_buf(),
101
            pool_config: PoolConfig::new(max(POOL_MINIMUM_SIZE, num_cpus::get_physical() * 4)),
102
            runtime_config: RuntimeConfig::default(),
103
            secret: None,
104
        }
105
    }
106
107
    /// Similar to [`SqliteStoreConfig::new`], but with defaults tailored for a
108
    /// low memory usage environment.
109
    ///
110
    /// The following defaults are set:
111
    ///
112
    /// * The `pool_max_size` is set to the number of physical CPU, so one
113
    ///   connection per physical thread,
114
    /// * The `cache_size` is set to 500Kib,
115
    /// * The `journal_size_limit` is set to 2Mib.
116
    pub fn with_low_memory_config<P>(path: P) -> Self
117
    where
118
        P: AsRef<Path>,
119
    {
120
        Self::new(path)
121
            // Maximum one connection per physical thread.
122
            .pool_max_size(num_cpus::get_physical())
123
            // Cache size is 500Kib.
124
            .cache_size(500_000)
125
            // Journal size limit is 2Mib.
126
            .journal_size_limit(2_000_000)
127
    }
128
129
    /// Override the path.
130
    pub fn path<P>(mut self, path: P) -> Self
131
    where
132
        P: AsRef<Path>,
133
    {
134
        self.path = path.as_ref().to_path_buf();
135
        self
136
    }
137
138
    /// Define the passphrase if the store is encoded.
139
    pub fn passphrase(mut self, passphrase: Option<&str>) -> Self {
140
        self.secret =
141
            passphrase.map(|passphrase| Secret::PassPhrase(Zeroizing::new(passphrase.to_owned())));
142
        self
143
    }
144
145
    /// Define the key if the store is encoded.
146
    pub fn key(mut self, key: Option<&[u8; 32]>) -> Self {
147
        self.secret = key.map(|key| Secret::Key(Box::new(*key)));
148
        self
149
    }
150
151
    /// Define the maximum pool size for [`deadpool`].
152
    ///
153
    /// See [`deadpool::managed::PoolConfig::max_size`] to learn more.
154
    pub fn pool_max_size(mut self, max_size: usize) -> Self {
155
        self.pool_config.max_size = max(POOL_MINIMUM_SIZE, max_size);
156
        self
157
    }
158
159
    /// Optimize the database.
160
    ///
161
    /// The SQLite documentation recommends to run this regularly and after any
162
    /// schema change. The easiest is to do it consistently when the store is
163
    /// constructed, after eventual migrations.
164
    ///
165
    /// See [`PRAGMA optimize`] to learn more.
166
    ///
167
    /// The default value is `true`.
168
    ///
169
    /// [`PRAGMA optimize`]: https://www.sqlite.org/pragma.html#pragma_optimize
170
    pub fn optimize(mut self, optimize: bool) -> Self {
171
        self.runtime_config.optimize = optimize;
172
        self
173
    }
174
175
    /// Define the maximum size in **bytes** the SQLite cache can use.
176
    ///
177
    /// See [`PRAGMA cache_size`] to learn more.
178
    ///
179
    /// The default value is 2Mib.
180
    ///
181
    /// [`PRAGMA cache_size`]: https://www.sqlite.org/pragma.html#pragma_cache_size
182
    pub fn cache_size(mut self, cache_size: u32) -> Self {
183
        self.runtime_config.cache_size = cache_size;
184
        self
185
    }
186
187
    /// Limit the size of the WAL file, in **bytes**.
188
    ///
189
    /// By default, while the DB connections of the databases are open, [the
190
    /// size of the WAL file can keep increasing][size_wal_file] depending on
191
    /// the size needed for the transactions. A critical case is `VACUUM`
192
    /// which basically writes the content of the DB file to the WAL file
193
    /// before writing it back to the DB file, so we end up taking twice the
194
    /// size of the database.
195
    ///
196
    /// By setting this limit, the WAL file is truncated after its content is
197
    /// written to the database, if it is bigger than the limit.
198
    ///
199
    /// See [`PRAGMA journal_size_limit`] to learn more. The value `limit`
200
    /// corresponds to `N` in `PRAGMA journal_size_limit = N`.
201
    ///
202
    /// The default value is 10Mib.
203
    ///
204
    /// [size_wal_file]: https://www.sqlite.org/wal.html#avoiding_excessively_large_wal_files
205
    /// [`PRAGMA journal_size_limit`]: https://www.sqlite.org/pragma.html#pragma_journal_size_limit
206
    pub fn journal_size_limit(mut self, limit: u32) -> Self {
207
        self.runtime_config.journal_size_limit = limit;
208
        self
209
    }
210
211
    /// Returns the pool configuration.
212
    pub(crate) fn pool_config(&self) -> PoolConfig {
213
        self.pool_config
214
    }
215
216
    /// Returns the runtime configuration.
217
    pub(crate) fn runtime_config(&self) -> RuntimeConfig {
218
        self.runtime_config
219
    }
220
221
    /// Build a pool of active connections to a particular database.
222
    pub fn build_pool_of_connections(
223
        &self,
224
        database_name: &str,
225
    ) -> Result<connection::Pool, connection::CreatePoolError> {
226
        let path = self.path.join(database_name);
227
        let manager = connection::Manager::new(path);
228
229
        connection::Pool::builder(manager)
230
            .config(self.pool_config)
231
            .runtime(connection::RUNTIME)
232
            .build()
233
            .map_err(connection::CreatePoolError::Build)
234
    }
235
}
236
237
/// This type represents values to set at runtime when a database is opened.
238
///
239
/// This configuration is applied by
240
/// [`utils::SqliteAsyncConnExt::apply_runtime_config`].
241
#[derive(Clone, Copy, Debug)]
242
struct RuntimeConfig {
243
    /// If `true`, [`utils::SqliteAsyncConnExt::optimize`] will be called.
244
    optimize: bool,
245
246
    /// Regardless of the value, [`utils::SqliteAsyncConnExt::cache_size`] will
247
    /// always be called with this value.
248
    cache_size: u32,
249
250
    /// Regardless of the value,
251
    /// [`utils::SqliteAsyncConnExt::journal_size_limit`] will always be called
252
    /// with this value.
253
    journal_size_limit: u32,
254
}
255
256
impl Default for RuntimeConfig {
257
    fn default() -> Self {
258
        Self {
259
            // Optimize is always applied.
260
            optimize: true,
261
            // A cache of 2Mib.
262
            cache_size: 2_000_000,
263
            // A limit of 10Mib.
264
            journal_size_limit: 10_000_000,
265
        }
266
    }
267
}
268
269
#[cfg(test)]
270
mod tests {
271
    use std::{
272
        ops::Not,
273
        path::{Path, PathBuf},
274
    };
275
276
    use super::{POOL_MINIMUM_SIZE, Secret, SqliteStoreConfig};
277
278
    #[test]
279
    fn test_new() {
280
        let store_config = SqliteStoreConfig::new(Path::new("foo"));
281
282
        assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical() * 4);
283
        assert!(store_config.runtime_config.optimize);
284
        assert_eq!(store_config.runtime_config.cache_size, 2_000_000);
285
        assert_eq!(store_config.runtime_config.journal_size_limit, 10_000_000);
286
    }
287
288
    #[test]
289
    fn test_with_low_memory_config() {
290
        let store_config = SqliteStoreConfig::with_low_memory_config(Path::new("foo"));
291
292
        assert_eq!(store_config.pool_config.max_size, num_cpus::get_physical());
293
        assert!(store_config.runtime_config.optimize);
294
        assert_eq!(store_config.runtime_config.cache_size, 500_000);
295
        assert_eq!(store_config.runtime_config.journal_size_limit, 2_000_000);
296
    }
297
298
    #[test]
299
    fn test_store_config_when_passphrase() {
300
        let store_config = SqliteStoreConfig::new(Path::new("foo"))
301
            .passphrase(Some("bar"))
302
            .pool_max_size(42)
303
            .optimize(false)
304
            .cache_size(43)
305
            .journal_size_limit(44);
306
307
        assert_eq!(store_config.path, PathBuf::from("foo"));
308
        assert_eq!(store_config.secret, Some(Secret::PassPhrase("bar".to_owned().into())));
309
        assert_eq!(store_config.pool_config.max_size, 42);
310
        assert!(store_config.runtime_config.optimize.not());
311
        assert_eq!(store_config.runtime_config.cache_size, 43);
312
        assert_eq!(store_config.runtime_config.journal_size_limit, 44);
313
    }
314
315
    #[test]
316
    fn test_store_config_when_key() {
317
        let store_config = SqliteStoreConfig::new(Path::new("foo"))
318
            .key(Some(&[
319
                143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61,
320
                107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156,
321
            ]))
322
            .pool_max_size(42)
323
            .optimize(false)
324
            .cache_size(43)
325
            .journal_size_limit(44);
326
327
        assert_eq!(store_config.path, PathBuf::from("foo"));
328
        assert_eq!(
329
            store_config.secret,
330
            Some(Secret::Key(Box::new([
331
                143, 27, 202, 78, 96, 55, 13, 149, 247, 8, 33, 120, 204, 92, 171, 66, 19, 238, 61,
332
                107, 132, 211, 40, 244, 71, 190, 99, 14, 173, 225, 6, 156,
333
            ])))
334
        );
335
        assert_eq!(store_config.pool_config.max_size, 42);
336
        assert!(store_config.runtime_config.optimize.not());
337
        assert_eq!(store_config.runtime_config.cache_size, 43);
338
        assert_eq!(store_config.runtime_config.journal_size_limit, 44);
339
    }
340
341
    #[test]
342
    fn test_store_config_path() {
343
        let store_config = SqliteStoreConfig::new(Path::new("foo")).path(Path::new("bar"));
344
345
        assert_eq!(store_config.path, PathBuf::from("bar"));
346
    }
347
348
    #[test]
349
    fn test_pool_size_has_a_minimum() {
350
        let store_config = SqliteStoreConfig::new(Path::new("foo")).pool_max_size(1);
351
352
        assert_eq!(store_config.pool_config.max_size, POOL_MINIMUM_SIZE);
353
    }
354
}