Commit 1ebff8f9 authored by Daniel García Moreno's avatar Daniel García Moreno

Move test lib to the mdl crate

parent fb190911
mdl/target
mdl/Cargo.lock
[package]
name = "lmdb_extras"
name = "mdl"
version = "0.1.0"
authors = ["Daniel García Moreno <danigm@wadobo.com>"]
......
extern crate lmdb;
extern crate failure;
extern crate serde;
extern crate bincode;
use std::path::Path;
use std::fs::create_dir_all;
use std::collections::HashMap;
use std::cell::RefCell;
use std::rc::Rc;
use failure::Error;
use failure::err_msg;
......@@ -22,18 +10,26 @@ use lmdb::WriteFlags;
use lmdb::RwCursor;
use lmdb::RoCursor;
use bincode::{serialize, deserialize};
use std::path::Path;
use std::fs::create_dir_all;
use std::collections::HashMap;
use std::cell::RefCell;
pub struct Continue(pub bool);
use store::Store;
use store::Continue;
/// Struct to store the lmdb environment
/// LMDB cache. This struct implements the Store trait so it can be used
/// to cache Model structs
#[derive(Debug)]
pub struct Cache {
/// LMDB environment
pub env: Environment,
/// database path in the filesystem
pub path: String,
dbs: HashMap<&'static str, Database>,
/// List of LMDB databases
dbs: RefCell<HashMap<&'static str, Database>>,
}
impl Cache {
......@@ -51,24 +47,29 @@ impl Cache {
Ok(Cache {
env: env,
path: path.to_string(),
dbs: HashMap::new(),
dbs: RefCell::new(HashMap::new()),
})
}
pub fn db(&mut self, name: &'static str) -> Result<Database, Error> {
if self.dbs.contains_key(name) {
return Ok(self.dbs[name].clone());
pub fn db(&self, name: &'static str) -> Result<Database, Error> {
// if the db is created, we return the db stored in cache
{
let dbs = self.dbs.borrow();
if dbs.contains_key(name) {
return Ok(dbs[name].clone());
}
}
// if the db doesn't exists, we create that db and store for the future
let db = self.env
.create_db(Some(name), DatabaseFlags::default())
.or(Err(err_msg(format!("error opening the db {}", name))))?;
self.dbs.insert(name, db.clone());
self.dbs.borrow_mut().insert(name, db.clone());
Ok(db)
}
pub fn rw<F, T>(&mut self, db: &'static str, op: F) -> Result<T, Error>
pub fn rw<F, T>(&self, db: &'static str, op: F) -> Result<T, Error>
where F: Fn(RwCursor) -> Result<T, Error> {
let db = self.db(db)?;
......@@ -83,7 +84,7 @@ impl Cache {
output
}
pub fn ro<F, T>(&mut self, db: &'static str, op: F) -> Result<T, Error>
pub fn ro<F, T>(&self, db: &'static str, op: F) -> Result<T, Error>
where F: Fn(RoCursor) -> Result<T, Error> {
let db = self.db(db)?;
......@@ -99,87 +100,31 @@ impl Cache {
}
}
pub trait Cacheable: serde::Serialize + serde::de::DeserializeOwned {
fn db() -> &'static str;
fn key(&self) -> String;
fn tob(&self) -> Result<Vec<u8>, Error> {
let encoded: Vec<u8> = serialize(self)?;
Ok(encoded)
}
fn fromb(data: &[u8]) -> Result<Self, Error> {
let decoded: Self = deserialize(data)?;
Ok(decoded)
}
/// Persist the struct in the database
fn store(&self, cache: &mut Cache) -> Result<(), Error> {
let key = self.key();
let data = self.tob()?;
cache.rw(Self::db(), move |mut cursor| {
cursor.put(&key.as_bytes(), &data, WriteFlags::empty())?;
Ok(())
})
}
/// Deletes the object from the database
fn delete(&self, cache: &mut Cache) -> Result<(), Error> {
let key = self.key();
cache.rw(Self::db(), move |mut cursor| {
cursor.get(Some(key.as_ref()), None, 15)?;
cursor.del(WriteFlags::empty())?;
impl Store for Cache {
fn push(&self, db: &'static str, key: &str, value: Vec<u8>)
-> Result<(), Error> {
self.rw(db, move |mut cursor| {
cursor.put(&key.as_bytes(), &value, WriteFlags::empty())?;
Ok(())
})
}
/// Loads the struct from the database
fn get(cache: &mut Cache, key: &str) -> Result<Self, Error> {
cache.ro(Self::db(), move |cursor| {
fn pull<F, T>(&self, db: &'static str, key: &str, formatter: F)
-> Result<T, Error>
where F: Fn(&[u8]) -> Result<T, Error> {
self.ro(db, move |cursor| {
let k = Some(key.as_ref());
let (_rkey, rdata) = cursor.get(k, None, 15)?;
Self::fromb(rdata)
formatter(rdata)
})
}
/// Iterates over all objects that starts with the same key
fn iter<F>(cache: &mut Cache, prefix: &str, f: F)
fn iter<F>(&self, db: &'static str, prefix: &str, f: F)
-> Result<(), Error>
where F: Fn(Self) -> Continue {
where F: Fn(&[u8]) -> Continue {
let l = prefix.len();
cache.ro(Self::db(), move |mut cursor| {
let k = Some(prefix.as_ref());
cursor.get(k, None, 17)?;
let mut iter = cursor.iter_from(k.unwrap())
.filter(|(k, _v)| { k.len() >= l && &k[0..l] == prefix[0..l].as_bytes() });
loop {
match iter.next() {
None => break,
Some((_, v)) => {
let obj = Self::fromb(v)?;
if let Continue(false) = f(obj) {
break;
}
}
}
}
Ok(())
})
}
/// Get all objects with this prefix
fn all(cache: &mut Cache, prefix: &str)
-> Result<Vec<Self>, Error> {
let l = prefix.len();
let output: Rc<RefCell<Vec<Self>>> = Rc::new(RefCell::new(vec![]));
let out = output.clone();
cache.ro(Self::db(), move |mut cursor| {
self.ro(db, move |mut cursor| {
let k = Some(prefix.as_ref());
cursor.get(k, None, 17)?;
......@@ -187,18 +132,24 @@ pub trait Cacheable: serde::Serialize + serde::de::DeserializeOwned {
.filter(|(k, _v)| { k.len() >= l && &k[0..l] == prefix[0..l].as_bytes() });
for (_, v) in iter {
let obj = Self::fromb(v);
if let Ok(obj) = obj {
out.borrow_mut().push(obj);
if let Continue(false) = f(v) {
break;
}
};
Ok(())
})?;
Ok(Rc::try_unwrap(output)
.map_err(|_| err_msg(format!("error reading from db")))?
.into_inner())
Ok(())
}
fn rm(&self, db: &'static str, key: &str) -> Result<(), Error> {
self.rw(db, move |mut cursor| {
cursor.get(Some(key.as_ref()), None, 15)?;
cursor.del(WriteFlags::empty())?;
Ok(())
})
}
}
extern crate lmdb;
extern crate failure;
extern crate serde;
extern crate bincode;
pub mod store;
pub mod cache;
pub mod model;
pub use store::Store;
pub use store::Continue;
pub use cache::Cache;
pub use model::Model;
use serde;
use failure::Error;
use bincode::{serialize, deserialize};
use store::Store;
use store::Continue;
/// Trait to implement Cacheable data Model
pub trait Model: serde::Serialize +
serde::de::DeserializeOwned {
/// key to identify the data object, this should be unique
fn key(&self) -> String;
/// database name, where to store instances of this struct
fn db() -> &'static str { "default" }
/// Data Struct serialization
fn tob(&self) -> Result<Vec<u8>, Error> {
let encoded: Vec<u8> = serialize(self)?;
Ok(encoded)
}
/// Data Struct deserialization
fn fromb(data: &[u8]) -> Result<Self, Error> {
let decoded: Self = deserialize(data)?;
Ok(decoded)
}
/// Persist the struct in the database
fn store<S: Store>(&self, store: &S) -> Result<(), Error> {
store.push(Self::db(), &self.key(), self.tob()?)
}
/// Deletes the object from the database
fn delete<S: Store>(&self, store: &S) -> Result<(), Error> {
store.rm(Self::db(), &self.key())
}
/// Loads the struct from the database
fn get<S: Store>(store: &S, key: &str) -> Result<Self, Error> {
store.pull(Self::db(), key, Self::fromb)
}
/// Get all objects with this prefix
fn all<S: Store>(store: &S, prefix: &str)
-> Result<Vec<Self>, Error> {
store.all(Self::db(), prefix, Self::fromb)
}
/// Iterate over all objects with this prefix
fn iter<S, F>(store: &S, prefix: &str, f: F) -> Result<(), Error>
where S: Store,
F: Fn(Self) -> Continue {
store.iter(Self::db(), prefix, move |data| {
match Self::fromb(data) {
Ok(obj) => f(obj),
_ => Continue(true)
}
})
}
}
use std::cell::RefCell;
use std::rc::Rc;
use failure::Error;
use failure::err_msg;
pub struct Continue(pub bool);
/// Trait that defines a Store that can be implemented to save Model objects
/// in memory, filesystem or the network
pub trait Store {
/// Stores the value in the database with the corresponding key
fn push(&self, db: &'static str, key: &str, value: Vec<u8>)
-> Result<(), Error>;
/// Retrieves the value in the database with the corresponding key
/// Returns an error if the key doesn't exists
fn pull<F, T>(&self, db: &'static str, key: &str, formatter: F)
-> Result<T, Error>
where F: Fn(&[u8]) -> Result<T, Error>;
/// Iterates over all objects that starts with the prefix and run
/// the function f. If f returns Continue(false) the iteration stops
fn iter<F>(&self, db: &'static str, prefix: &str, f: F)
-> Result<(), Error>
where F: Fn(&[u8]) -> Continue;
/// Retrieves all items in the database that starts with the prefix key
fn all<F, T>(&self, db: &'static str, prefix: &str, formatter: F)
-> Result<Vec<T>, Error>
where F: Fn(&[u8]) -> Result<T, Error> {
let output: Rc<RefCell<Vec<T>>> = Rc::new(RefCell::new(vec![]));
let out = output.clone();
self.iter(db, prefix, move |data| {
if let Ok(obj) = formatter(data) {
out.borrow_mut().push(obj);
}
Continue(true)
})?;
Ok(Rc::try_unwrap(output)
.map_err(|_| err_msg(format!("error reading from db")))?
.into_inner())
}
/// Remove the corresponding data in the database by key
fn rm(&self, db: &'static str, key: &str) -> Result<(), Error>;
}
extern crate lmdb_extras;
extern crate mdl;
#[macro_use]
extern crate serde_derive;
use lmdb_extras::Cache;
use lmdb_extras::Cacheable;
use lmdb_extras::Continue;
use mdl::Cache;
use mdl::Model;
use mdl::Continue;
use std::fs::remove_dir_all;
use std::cell::RefCell;
use std::rc::Rc;
static DB: &'static str = "/tmp/test.lmdb";
......@@ -19,8 +17,7 @@ struct A {
pub p1: String,
pub p2: u32,
}
impl Cacheable for A {
fn db() -> &'static str { "TESTDB" }
impl Model for A {
fn key(&self) -> String {
format!("{}:{}", self.p1, self.p2)
}
......@@ -31,8 +28,7 @@ struct B {
pub id: u32,
pub complex: Vec<String>,
}
impl Cacheable for B {
fn db() -> &'static str { "TESTDB" }
impl Model for B {
fn key(&self) -> String {
format!("b:{}", self.id)
}
......@@ -42,13 +38,13 @@ impl Cacheable for B {
#[test]
fn basic_struct_test() {
let db = &format!("{}-basic", DB);
let mut cache = Cache::new(db).unwrap();
let cache = Cache::new(db).unwrap();
let a = A{ p1: "hello".to_string(), p2: 42 };
let r = a.store(&mut cache);
let r = a.store(&cache);
assert!(r.is_ok());
let a1: A = A::get(&mut cache, "hello:42").unwrap();
let a1: A = A::get(&cache, "hello:42").unwrap();
assert_eq!(a1.p1, a.p1);
assert_eq!(a1.p2, a.p2);
let _ = remove_dir_all(db);
......@@ -57,19 +53,19 @@ fn basic_struct_test() {
#[test]
fn delete_test() {
let db = &format!("{}-delete", DB);
let mut cache = Cache::new(db).unwrap();
let cache = Cache::new(db).unwrap();
let a = A{ p1: "hello".to_string(), p2: 42 };
let r = a.store(&mut cache);
let r = a.store(&cache);
assert!(r.is_ok());
let r = A::get(&mut cache, "hello:42");
let r = A::get(&cache, "hello:42");
assert!(r.is_ok());
let r = a.delete(&mut cache);
let r = a.delete(&cache);
assert!(r.is_ok());
let r = A::get(&mut cache, "hello:42");
let r = A::get(&cache, "hello:42");
assert!(r.is_err());
let _ = remove_dir_all(db);
}
......@@ -77,57 +73,45 @@ fn delete_test() {
#[test]
fn iterate_test() {
let db = &format!("{}-it", DB);
let mut cache = Cache::new(db).unwrap();
let cache = Cache::new(db).unwrap();
for i in 1..10 {
let a = A{ p1: "hello".to_string(), p2: i };
let r = a.store(&mut cache);
let r = a.store(&cache);
assert!(r.is_ok());
}
//inserting other objects in cache
for i in 1..10 {
let b = B{ id: i, complex: vec![] };
let r = b.store(&mut cache);
let r = b.store(&cache);
assert!(r.is_ok());
}
//and now more A objects
for i in 10..20 {
let a = A{ p1: "hello".to_string(), p2: i };
let r = a.store(&mut cache);
let r = a.store(&cache);
assert!(r.is_ok());
}
let r = A::get(&mut cache, "hello:1");
let r = A::get(&cache, "hello:1");
assert!(r.is_ok());
assert_eq!(r.unwrap().p2, 1);
let r = B::get(&mut cache, "b:1");
let r = B::get(&cache, "b:1");
assert!(r.is_ok());
assert_eq!(r.unwrap().id, 1);
// Iterate over all A elements
let v = Rc::new(RefCell::new(vec![]));
A::iter(&mut cache, "hello", |a| {
v.borrow_mut().push(a);
Continue(true)
}).unwrap();
let mut v = v.borrow_mut();
let mut v = A::all(&cache, "hello").unwrap();
v.sort_by_key(|a| a.p2);
for (i, a) in v.iter().enumerate() {
assert_eq!(a.p2, (i+1) as u32);
}
// Iterate over all B elements
let v = Rc::new(RefCell::new(vec![]));
B::iter(&mut cache, "b", |b| {
v.borrow_mut().push(b);
Continue(true)
}).unwrap();
let mut v = v.borrow_mut();
let mut v = B::all(&cache, "b").unwrap();
v.sort_by_key(|b| b.id);
for (i, b) in v.iter().enumerate() {
assert_eq!(b.id, (i+1) as u32);
......@@ -139,30 +123,29 @@ fn iterate_test() {
#[test]
fn iterate_write_test() {
let db = &format!("{}-it2", DB);
let mut cache = Cache::new(db).unwrap();
let cache = Cache::new(db).unwrap();
//inserting other objects in cache
for i in 1..10 {
let b = B{ id: i, complex: vec![] };
let r = b.store(&mut cache);
let r = b.store(&cache);
assert!(r.is_ok());
}
// Iterate over all B elements
let all = B::all(&mut cache, "b").unwrap();
let all = B::all(&cache, "b").unwrap();
for mut b in all {
b.complex.push("UPDATED".to_string());
b.store(&mut cache).unwrap();
b.store(&cache).unwrap();
}
// Iterate over all B elements
B::iter(&mut cache, "b", |b| {
println!("{:?}", b.complex);
B::iter(&cache, "b", |b| {
assert_eq!(b.complex.len(), 1);
Continue(true)
}).unwrap();
let _ = remove_dir_all(db);
}
......@@ -171,25 +154,25 @@ fn thread_test() {
use std::thread;
let db = &format!("{}-thread", DB);
let mut cache = Cache::new(db).unwrap();
let cache = Cache::new(db).unwrap();
let b = B{ id: 1, complex: vec![] };
let _ = b.store(&mut cache);
let _ = b.store(&cache);
let join_handle: thread::JoinHandle<_> =
thread::spawn(move || {
let db = &format!("{}-thread", DB);
let mut cache = Cache::new(db).unwrap();
let mut b = B::get(&mut cache, "b:1").unwrap();
let cache = Cache::new(db).unwrap();
let mut b = B::get(&cache, "b:1").unwrap();
assert_eq!(b.complex.len(), 0);
b.complex.push("modified".to_string());
let _ = b.store(&mut cache);
let _ = b.store(&cache);
});
// waiting for the thread to finish
join_handle.join().unwrap();
let b = B::get(&mut cache, "b:1").unwrap();
let b = B::get(&cache, "b:1").unwrap();
assert_eq!(b.id, 1);
assert_eq!(b.complex.len(), 1);
assert_eq!(&b.complex[0][..], "modified");
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment