Verified Commit abc8fb98 authored by Jordan Petridis's avatar Jordan Petridis 🌱

Pipeline: Dispatch feed indexing to the rayon threadpool.

parent 885c525d
Pipeline #9116 failed with stages
in 2 minutes and 40 seconds
...@@ -690,6 +690,7 @@ dependencies = [ ...@@ -690,6 +690,7 @@ dependencies = [
"pretty_assertions 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_assertions 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon-futures 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"rfc822_sanitizer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "rfc822_sanitizer 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)",
"rss 1.4.0 (git+https://github.com/alatiera/rss)", "rss 1.4.0 (git+https://github.com/alatiera/rss)",
"string_cache 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", "string_cache 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)",
...@@ -1407,6 +1408,15 @@ dependencies = [ ...@@ -1407,6 +1408,15 @@ dependencies = [
"rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "rayon-futures"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"futures 0.1.21 (registry+https://github.com/rust-lang/crates.io-index)",
"rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.1.37" version = "0.1.37"
...@@ -2263,6 +2273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" ...@@ -2263,6 +2273,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
"checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5" "checksum rand 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "eba5f8cb59cc50ed56be8880a5c7b496bfd9bd26394e176bc67884094145c2c5"
"checksum rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80e811e76f1dbf68abf87a759083d34600017fc4e10b6bd5ad84a700f9dba4b1" "checksum rayon 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "80e811e76f1dbf68abf87a759083d34600017fc4e10b6bd5ad84a700f9dba4b1"
"checksum rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d24ad214285a7729b174ed6d3bcfcb80177807f959d95fafd5bfc5c4f201ac8" "checksum rayon-core 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9d24ad214285a7729b174ed6d3bcfcb80177807f959d95fafd5bfc5c4f201ac8"
"checksum rayon-futures 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ea70dae42033c388536c7e9bdca084a14d75659c12e25fc105e7552464a87c4a"
"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd" "checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd"
"checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76"
"checksum regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "aec3f58d903a7d2a9dc2bf0e41a746f4530e0cab6b615494e058f67a3ef947fb" "checksum regex 0.2.10 (registry+https://github.com/rust-lang/crates.io-index)" = "aec3f58d903a7d2a9dc2bf0e41a746f4530e0cab6b615494e058f67a3ef947fb"
......
...@@ -11,6 +11,7 @@ derive_builder = "0.5.1" ...@@ -11,6 +11,7 @@ derive_builder = "0.5.1"
lazy_static = "1.0.0" lazy_static = "1.0.0"
log = "0.4.1" log = "0.4.1"
rayon = "1.0.1" rayon = "1.0.1"
rayon-futures = "0.1.0"
rfc822_sanitizer = "0.3.3" rfc822_sanitizer = "0.3.3"
rss = "1.4.0" rss = "1.4.0"
url = "1.7.0" url = "1.7.0"
......
...@@ -51,6 +51,7 @@ extern crate hyper_tls; ...@@ -51,6 +51,7 @@ extern crate hyper_tls;
extern crate native_tls; extern crate native_tls;
extern crate num_cpus; extern crate num_cpus;
extern crate rayon; extern crate rayon;
extern crate rayon_futures;
extern crate rfc822_sanitizer; extern crate rfc822_sanitizer;
extern crate rss; extern crate rss;
extern crate tokio_core; extern crate tokio_core;
......
...@@ -13,6 +13,8 @@ use hyper_tls::HttpsConnector; ...@@ -13,6 +13,8 @@ use hyper_tls::HttpsConnector;
use tokio_core::reactor::Core; use tokio_core::reactor::Core;
use num_cpus; use num_cpus;
use rayon;
use rayon_futures::ScopeFutureExt;
use rss; use rss;
use dbqueries; use dbqueries;
...@@ -59,7 +61,7 @@ where ...@@ -59,7 +61,7 @@ where
{ {
let pipeline = sources let pipeline = sources
.and_then(clone!(client => move |s| s.into_feed(client.clone(), ignore_etags))) .and_then(clone!(client => move |s| s.into_feed(client.clone(), ignore_etags)))
.and_then(|feed| feed.index()) .and_then(|feed| rayon::scope(|s| s.spawn_future(feed.index())))
// the stream will stop at the first error so // the stream will stop at the first error so
// we ensure that everything will succeded regardless. // we ensure that everything will succeded regardless.
.map_err(|err| error!("Error: {}", err)) .map_err(|err| error!("Error: {}", err))
......
...@@ -29,7 +29,8 @@ ...@@ -29,7 +29,8 @@
], ],
"env" : { "env" : {
"CARGO_HOME" : "/run/build/Hammond/cargo", "CARGO_HOME" : "/run/build/Hammond/cargo",
"RUST_BACKTRACE" : "1" "RUST_BACKTRACE" : "1",
"RUSTFLAGS" : "--cfg rayon_unstable"
} }
}, },
"modules" : [ "modules" : [
......
#!/bin/sh #!/bin/sh
export CARGO_HOME=$1/target/cargo-home export CARGO_HOME=$1/target/cargo-home
export RUSTFLAGS=--cfg rayon_unstable
cargo build --release -p hammond-gtk && cp $1/target/release/hammond-gtk $2 cargo build --release -p hammond-gtk && cp $1/target/release/hammond-gtk $2
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment