From 0b3e6767864d3c1c7d698d3d4f383de0ae4084b6 Mon Sep 17 00:00:00 2001 From: Katharina Fey Date: Tue, 4 Sep 2018 16:29:03 +0200 Subject: Cleaning up the core a bit. Adding better (any) config handling to files --- Cargo.lock | 13 ++++ lockchain-core/src/traits.rs | 8 ++- lockchain-core/src/users/mod.rs | 3 +- lockchain-core/src/users/tokens.rs | 1 - lockchain-core/src/users/userstore.rs | 13 ++-- lockchain-files/Cargo.toml | 7 +++ lockchain-files/examples/create.rs | 60 +++++++++--------- lockchain-files/examples/register_user.rs | 0 lockchain-files/src/config.rs | 85 +++++++++++++++++++++++++ lockchain-files/src/fs.rs | 81 +++++++++++------------- lockchain-files/src/lib.rs | 100 ++++++++++++++++++++++++++++-- lockchain-files/src/utils.rs | 26 ++++++++ 12 files changed, 306 insertions(+), 91 deletions(-) delete mode 100644 lockchain-files/examples/register_user.rs create mode 100644 lockchain-files/src/config.rs create mode 100644 lockchain-files/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 096e582..f308df4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -962,6 +962,10 @@ name = "lockchain-files" version = "0.9.1-alpha.0" dependencies = [ "lockchain-core 0.9.1-alpha.0", + "semver 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1869,6 +1873,14 @@ dependencies = [ "tokio-reactor 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "toml" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.75 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "trust-dns-proto" version = "0.3.3" @@ -2309,6 +2321,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum tokio-timer 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "d03fa701f9578a01b7014f106b47f0a363b4727a7f3f75d666e312ab7acbbf1c" "checksum tokio-udp 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "da941144b816d0dcda4db3a1ba87596e4df5e860a72b70783fe435891f80601c" "checksum tokio-uds 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "424c1ed15a0132251813ccea50640b224c809d6ceafb88154c1a8775873a0e89" +"checksum toml 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a0263c6c02c4db6c8f7681f9fd35e90de799ebd4cfdeab77a38f4ff6b3d8c0d9" "checksum trust-dns-proto 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "cbbddb93547eeee847367d8f59b68002294a7b4df31c143fbee4109ce0c61a04" "checksum trust-dns-resolver 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9b0a0c9d4f8dd56481209c5ae1a8965ed022461d352c81fb92466ec9d846929e" "checksum typenum 1.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "612d636f949607bdf9b123b4a6f6d966dedf3ff669f7f045890d3a4a73948169" diff --git a/lockchain-core/src/traits.rs b/lockchain-core/src/traits.rs index 14f8310..58b63b2 100644 --- a/lockchain-core/src/traits.rs +++ b/lockchain-core/src/traits.rs @@ -125,7 +125,7 @@ where /// A shared constructor for all vault implementations fn new(name: &str, location: &str) -> Self; /// Load and open an existing vault - fn load(name: &str, location: &str) -> Self; + fn load(name: &str, location: &str) -> Option>; /// Unlock the vault for a specific user fn authenticate(&mut self, username: &str, secret: &str) -> Token; /// End a specific user session @@ -138,6 +138,12 @@ where /// Pull a specific record from the backend fn pull(&mut self, name: &str); /// Sync all changes back to the backend + /// + /// Ultimately it's up to the backend to decide + /// how changes are synced. + /// It's free to ignore any sync requests + /// but they can still be made for backends + /// which explicitly promise sync requests compliance fn sync(&mut self); /// Get a complete record from this vault diff --git a/lockchain-core/src/users/mod.rs b/lockchain-core/src/users/mod.rs index bad4256..6482264 100644 --- a/lockchain-core/src/users/mod.rs +++ b/lockchain-core/src/users/mod.rs @@ -21,7 +21,7 @@ mod tokens; mod keystore; mod secrets; -mod userstore; +pub mod userstore; pub use self::auth::pam_authenticate; pub use self::keystore::KeyStore; @@ -31,7 +31,6 @@ pub use self::user::User; pub use errors::AuthError; pub use self::rights::{Access, Role}; -use crypto::{encoding, hashing, random}; use std::collections::HashMap; use { meta::MetaDomain, diff --git a/lockchain-core/src/users/tokens.rs b/lockchain-core/src/users/tokens.rs index 32a6e01..f7226ef 100644 --- a/lockchain-core/src/users/tokens.rs +++ b/lockchain-core/src/users/tokens.rs @@ -1,5 +1,4 @@ use crypto::random; -use traits::AutoEncoder; const TOK_SIZE: usize = 64; diff --git a/lockchain-core/src/users/userstore.rs b/lockchain-core/src/users/userstore.rs index 60fee0b..483c4ee 100644 --- a/lockchain-core/src/users/userstore.rs +++ b/lockchain-core/src/users/userstore.rs @@ -1,13 +1,14 @@ +//! Merging `KeyStore` and `Userstore` into the same concept + use super::rights::Access; -use super::secrets::SecretType; use crypto::Key; use std::collections::HashMap; -/// A thin user keystore +/// A thin user UserStore /// /// It's implementation can manage multiple keys per user, of various /// types and constrained for limited access rights. -pub struct KeyStore { +pub struct UserStore { store: HashMap, } @@ -16,11 +17,11 @@ struct StoreUser { keys: HashMap, } -impl KeyStore { - /// Create a new, empty keystore +impl UserStore { + /// Create a new, empty UserStore /// /// This is most likely *not* what you want. Instead, transform - /// a `MetaData` object into a keystore. + /// a `MetaData` object into a UserStore. pub fn new() -> Self { Self { store: HashMap::new(), diff --git a/lockchain-files/Cargo.toml b/lockchain-files/Cargo.toml index e63455a..a4b5086 100644 --- a/lockchain-files/Cargo.toml +++ b/lockchain-files/Cargo.toml @@ -9,5 +9,12 @@ homepage = "https://github.com/spacekookie/lockchain/tree/master/lockchain-files readme = "README.md" license = "MIT/X11 OR Apache-2.0" +# Specifies a semver vault version it can open +vault-version = "0.1" + [dependencies] lockchain-core = { version = "0.9.1-alpha.0", path = "../lockchain-core" } +semver = "0.9.0" +toml = "0.4" +serde = "1.0" +serde_derive = "1.0" \ No newline at end of file diff --git a/lockchain-files/examples/create.rs b/lockchain-files/examples/create.rs index 4b19180..e9a4994 100644 --- a/lockchain-files/examples/create.rs +++ b/lockchain-files/examples/create.rs @@ -1,36 +1,36 @@ -extern crate lockchain_core as lcc; -extern crate lockchain_files as files; +// extern crate lockchain_core as lcc; +// extern crate lockchain_files as files; -use files::DataVault; -use lcc::traits::Vault; -use lcc::users::{User, UserStore}; -use lcc::{EncryptedBody, Payload, Record}; -use std::env; +// use files::DataVault; +// use lcc::traits::Vault; +// use lcc::users::{User, UserStore}; +// use lcc::{EncryptedBody, Payload, Record}; +// use std::env; -fn main() { - if env::args().len() == 3 { - let path = env::args().nth(1).unwrap(); - let name = env::args().nth(2).unwrap(); +// fn main() { +// if env::args().len() == 3 { +// let path = env::args().nth(1).unwrap(); +// let name = env::args().nth(2).unwrap(); - let mut vault: DataVault = DataVault::new(&name, &path); - let mut store = match ( - vault.meta_pull_domain("userstore"), - vault.meta_pull_domain("registry"), - ) { - (Some(users), Some(registry)) => (users.clone(), registry.clone()).into(), - _ => UserStore::default(), - }; +// let mut vault: DataVault = DataVault::new(&name, &path); +// let mut store = match ( +// vault.meta_pull_domain("userstore"), +// vault.meta_pull_domain("registry"), +// ) { +// (Some(users), Some(registry)) => (users.clone(), registry.clone()).into(), +// _ => UserStore::default(), +// }; - /* Some users of our vault have the same password :S */ - store.add(User::register("alice", "password")); - let token = store.get_token(vec!()); +// /* Some users of our vault have the same password :S */ +// store.add(User::register("alice", "password")); +// let token = store.get_token(vec!()); - let (users, registry) = store.into(); +// let (users, registry) = store.into(); - vault.meta_push_domain(users); - vault.meta_push_domain(registry); - vault.sync(); - } else { - eprintln!("Usage: create [FLAGS] (there are no flags)") - } -} +// vault.meta_push_domain(users); +// vault.meta_push_domain(registry); +// vault.sync(); +// } else { +// eprintln!("Usage: create [FLAGS] (there are no flags)") +// } +// } diff --git a/lockchain-files/examples/register_user.rs b/lockchain-files/examples/register_user.rs deleted file mode 100644 index e69de29..0000000 diff --git a/lockchain-files/src/config.rs b/lockchain-files/src/config.rs new file mode 100644 index 0000000..b2d311a --- /dev/null +++ b/lockchain-files/src/config.rs @@ -0,0 +1,85 @@ +use std::error::Error; +use std::fmt; +use std::time::SystemTime; +use std::{fs::File, path::PathBuf, io}; + +use semver::Version; +use toml; +use utils::FileToString; + +/// A set of errors around `lockchain-files` configs +#[derive(Debug)] +pub enum ConfigError { + IncompatibleVersion(String, String), + ConfigCorrupted, +} + +impl fmt::Display for ConfigError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use self::ConfigError::*; + write!( + f, + "{}", + match self { + IncompatibleVersion(f, l) => { + format!("Version '{}' is incompatible with library '{}'", f, l) + } + ConfigCorrupted => "Configuration file was corrupted!".into(), + } + ) + } +} + +impl Error for ConfigError {} + +/// The configuration describing a file vault +#[derive(Serialize, Deserialize)] +pub struct VaultConfig { + /// A semver conforming version string + pub version: String, + pub created_at: SystemTime, + pub modified_at: SystemTime, +} + +impl VaultConfig { + pub fn new() -> Self { + Self { + version: "0.1".into(), + created_at: SystemTime::now(), + modified_at: SystemTime::now(), + } + } + + pub fn save(&self) -> Result<(), io::Error> { + + Ok(()) + } + + /// Attempts to load a configuration – returning detailed errors + pub fn load(vault: &PathBuf) -> Result { + let mut cfg_path = vault.clone(); + cfg_path.push("config.toml"); + + let cfg: VaultConfig = match File::open(cfg_path.as_path()) { + Ok(mut f) => match f.get_string() { + Ok(s) => match toml::from_str(&s) { + Ok(c) => c, + Err(_) => return Err(ConfigError::ConfigCorrupted), + }, + Err(_) => return Err(ConfigError::ConfigCorrupted), + }, + Err(_) => return Err(ConfigError::ConfigCorrupted), + }; + + let version = match Version::parse(&cfg.version) { + Ok(v) => v, + Err(_) => return Err(ConfigError::ConfigCorrupted), + }; + + if version != Version::parse("0.1").unwrap() { + Err(ConfigError::IncompatibleVersion(cfg.version, "0.1".into())) + } else { + Ok(cfg) + } + } +} diff --git a/lockchain-files/src/fs.rs b/lockchain-files/src/fs.rs index be5ccba..1aa57bc 100644 --- a/lockchain-files/src/fs.rs +++ b/lockchain-files/src/fs.rs @@ -1,10 +1,10 @@ //! Filesystem abstraction for various data types -//! +//! //! All operations return io::Result<()> to indicate errors //! and functions that have multiple file endpoints will return //! a folded error list to indicate which ops were successful //! and which failed. -//! +//! //! There is also a `From> for Result` implementation //! which will return either `Ok(())` or the first error in the list //! of operations. @@ -13,20 +13,23 @@ use lcc::traits::AutoEncoder; use std::collections::HashMap; use std::error::Error; -use std::io::{self, Read, Write}; +use std::io::Write; use std::{ - fs::{self, File, OpenOptions as OO}, path::PathBuf, + fs::{self, File, OpenOptions as OO}, + path::PathBuf, }; +use utils::FileToString; + #[derive(Debug)] pub struct Filesystem { - name: String, - path: String, - root: PathBuf, + pub name: String, + pub path: String, + pub root: PathBuf, } /// A switching enum to determine what type of file to load - +#[allow(dead_code)] pub enum FileType { /// A data record file Record, @@ -34,6 +37,8 @@ pub enum FileType { Metadata, /// A simple checksum file Checksum, + #[doc(hidden)] + __NonExhaustive, } /// Construct a file ending for a specific match result @@ -42,18 +47,22 @@ macro_rules! file_ending { match $type { FileType::Record => "record", FileType::Metadata => "meta", + FileType::Checksum => "sum", _ => "dat", } }; } impl Filesystem { - pub fn create(path: &str, name: &str) -> Filesystem { + /// Create a new filesystem representation + /// + /// This function does _not_ touch the disk! + pub fn new(path: &str, name: &str) -> Self { let mut buffer = PathBuf::new(); buffer.push(path); buffer.push(format!("{}.vault", name)); - Filesystem { + Self { name: name.to_owned(), path: path.to_owned(), root: buffer, @@ -74,27 +83,29 @@ impl Filesystem { Ok(fs::read_dir(match types { FileType::Record => self.root.join("records"), FileType::Metadata => self.root.join("metadata"), - _ => self.root.join("."), + _ => self.root.clone(), })?.into_iter() - .filter_map(|r| r.ok()) - .filter(|f| match f.file_type() { - Ok(vf) => vf.is_file(), - _ => false, - }) - .map(|de| de.path()) - .filter_map(|p| p.into_os_string().into_string().ok()) - .filter_map(|s| File::open(s).ok()) - .filter_map(|mut f| f.get_string().ok()) - .filter_map(|s| T::decode(&s).ok()) - .collect()) + .filter_map(|r| r.ok()) + .filter(|f| match f.file_type() { + Ok(vf) => vf.is_file(), + _ => false, + }).map(|de| de.path()) + .filter_map(|p| p.into_os_string().into_string().ok()) + .filter_map(|s| File::open(s).ok()) + .filter_map(|mut f| f.get_string().ok()) + .filter_map(|s| T::decode(&s).ok()) + .collect()) } + /// Retrieve a single record from the cached vault pub fn pull(&self, types: FileType, id: &str) -> Result> { Ok(T::decode( - &File::open(self.root.join(&format!("{}.{}", id, file_ending!(types))))?.get_string()?, + &File::open(self.root.join(&format!("{}.{}", id, file_ending!(types))))? + .get_string()?, )?) } + /// Respond to a sync request pub fn sync(&self, data: &HashMap, types: FileType) -> Result<(), Box> where T: AutoEncoder, @@ -110,13 +121,11 @@ impl Filesystem { }.join(format!("{}.{}", k, file_ending!(types))), v, ) - }) - .filter(|(_, v)| v.is_some()) + }).filter(|(_, v)| v.is_some()) .map(|(k, v)| (k, v.unwrap())) .map(|(path, data): (PathBuf, String)| { (OO::new().create(true).write(true).open(path), data) - }) - .filter(|(path, _)| path.is_ok()) + }).filter(|(path, _)| path.is_ok()) .map(|(file, data)| (file.unwrap(), data)) .for_each(|(mut file, data)| { file.write_all(data.as_bytes()) @@ -126,21 +135,3 @@ impl Filesystem { Ok(()) } } - -/// A utility trait to read the conents from a file in -/// a single line. -pub trait FileToString { - /// Read the file contents into a string without any - /// error handling. - fn get_string(&mut self) -> Result; -} - -impl FileToString for File { - fn get_string(&mut self) -> Result { - let mut s = String::new(); - return match self.read_to_string(&mut s) { - Ok(_) => Ok(s), - Err(e) => Err(e), - }; - } -} diff --git a/lockchain-files/src/lib.rs b/lockchain-files/src/lib.rs index afa3a87..965f669 100644 --- a/lockchain-files/src/lib.rs +++ b/lockchain-files/src/lib.rs @@ -1,18 +1,87 @@ -//! A module that enables file management for vaults +//! A persistence layer for lockchain vaults based on files //! +//! This crate provides a filesystem backend +//! which relies on keeping records in discrete files +//! and folder structures. //! +//! This is great if a vault needs to be easily syncable +//! or indexable by another tool. +//! No clear-text secrets are ever written to disk. +//! But can sometimes be held in memory cache +//! for a period of time. +//! +//! This backend is comparibly slow +//! and should be avoided +//! for performance critical applications. +//! For such applications +//! the blockstore is much more suited +//! which represents a vault +//! in a binary blob +//! independant of the used filesystem. +//! +//! Part of the performance problems +//! comes from locking the entire vault +//! when doing operations, +//! meaning that only +//! one instance +//! of a lockchain library +//! can operate on it +//! at the time +//! +//! ``` +//! my_vault/ +//! config.toml +//! Lockfile +//! metadata/ +//! userstore.meta +//! registry.meta +//! records/ +//! .rec +//! .rec +//! .rec +//! hashsums/ +//! .sum +//! .sum +//! .sum +//! ``` #![feature(non_modrs_mods)] extern crate lockchain_core as lcc; +extern crate semver; +extern crate toml; + +#[macro_use] +extern crate serde_derive; +extern crate serde; use lcc::traits::{Body, LoadRecord, Vault}; use lcc::{users::Token, MetaDomain, Payload, Record, VaultMetadata}; use std::collections::HashMap; mod fs; -use fs::{FileType, Filesystem}; +mod utils; +mod config; -/// Represents a vault on disk +use fs::{FileType, Filesystem}; +pub use config::{VaultConfig, ConfigError}; + + +/// Persistence mapper to a folder and file structure +/// +/// Implements the `Vault` API in full, +/// replicating all functionality in memory +/// and never writing clear text data to disk. +/// +/// The internal layout should not be assumed +/// and isn't stabilised with the crate version +/// (i.e. minor crate bumps can break vault compatibility +/// as long as they remain API compatible). +/// +/// The version of a vault is written in it's coniguration +/// (which won't change – ever). +/// +/// The vault folder is safe to copy around – +/// all vault metadata is kept inside it. #[derive(Debug)] pub struct DataVault { meta_info: (String, String), @@ -27,6 +96,15 @@ impl DataVault { self.fs.scaffold(); self } + + fn load(mut self) -> Option> { + let config = match VaultConfig::load(&self.fs.root) { + Ok(cfg) => cfg, + _ => return None, + }; + + Some(Box::new(self)) + } } impl LoadRecord for DataVault {} @@ -37,12 +115,22 @@ impl Vault for DataVault { meta_info: (name.into(), location.into()), records: HashMap::new(), metadata: HashMap::new(), - fs: Filesystem::create(location, name), + fs: Filesystem::new(location, name), }.initialize() } - fn load(name: &str, location: &str) -> Self { - unimplemented!() + // Checking if a vault exists is basically checking it's config + // against the compatible version of this library. + // + // If it's compatible we can open the vault into memory + // (loading all required paths into the struct), then return it + fn load(name: &str, location: &str) -> Option> { + Self { + meta_info: (name.into(), location.into()), + records: HashMap::new(), + metadata: HashMap::new(), + fs: Filesystem::new(location, name), + }.load() } fn authenticate(&mut self, username: &str, secret: &str) -> Token { diff --git a/lockchain-files/src/utils.rs b/lockchain-files/src/utils.rs new file mode 100644 index 0000000..c724d32 --- /dev/null +++ b/lockchain-files/src/utils.rs @@ -0,0 +1,26 @@ +//! Small utility module for file operations + +use std::io::{self, Read}; +use std::fs::File; + +pub fn check_config() { + +} + +/// A utility trait to read the conents from a file in +/// a single line. +pub trait FileToString { + /// Read the file contents into a string without any + /// error handling. + fn get_string(&mut self) -> Result; +} + +impl FileToString for File { + fn get_string(&mut self) -> Result { + let mut s = String::new(); + return match self.read_to_string(&mut s) { + Ok(_) => Ok(s), + Err(e) => Err(e), + }; + } +} -- cgit v1.2.3