Merge pull request 'threads' (#9) from threads into master

Reviewed-on: #9

Cargo.lock

@@ -48,6 +48,7 @@ dependencies = [
  "indexmap",
  "lazy_static",
  "log",
+ "rayon",
  "serde",
  "serde_derive",
  "serde_json",
@@ -86,6 +87,57 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "crossbeam-channel"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
+dependencies = [
+ "cfg-if",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
+dependencies = [
+ "cfg-if",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "lazy_static",
+]
+
+[[package]]
+name = "either"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
+
 [[package]]
 name = "encoding_rs"
 version = "0.8.28"
@@ -139,6 +191,7 @@ checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
 dependencies = [
  "autocfg",
  "hashbrown",
+ "rayon",
  "serde",
 ]
 
@@ -184,6 +237,15 @@ version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
 
+[[package]]
+name = "memoffset"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
+dependencies = [
+ "autocfg",
+]
+
 [[package]]
 name = "num-integer"
 version = "0.1.44"
@@ -203,6 +265,16 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.7.2"
@@ -233,6 +305,31 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rayon"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
+dependencies = [
+ "autocfg",
+ "crossbeam-deque",
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
+dependencies = [
+ "crossbeam-channel",
+ "crossbeam-deque",
+ "crossbeam-utils",
+ "lazy_static",
+ "num_cpus",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -267,6 +364,12 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
+[[package]]
+name = "scopeguard"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+
 [[package]]
 name = "sct"
 version = "0.6.0"

Cargo.toml

@@ -5,9 +5,10 @@ name = "cggitem_sets"
 version = "0.1.0"
 
 [dependencies]
-indexmap = {version = "1.6", features = ["serde-1"]}
+indexmap = {version = "1.6", features = ["serde-1", "rayon"]}
 lazy_static = "1.4"
 log = "0.4"
+rayon = "1.5"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = {version = "1.0", features = ["preserve_order"]}

src/data_source.rs

@@ -62,7 +62,7 @@ pub trait DataSource {
         path: &PathBuf,
         client: &ureq::Agent,
     ) {
-        info!("Retrieving data for {} at {}", champ.name, pos);
+        info!("{}: Retrieving data for {} at {}", self.get_alias(), champ.name, pos);
         let data = self.get_champ_data_with_win_pourcentage(champ, pos, client);
 
         match data {
@@ -77,7 +77,7 @@ pub trait DataSource {
                     blocks: data.0,
                 };
 
-                info!("Writing item set for {} at {}", champ.name, pos);
+                info!("{}: Writing item set for {} at {}", self.get_alias(), champ.name, pos);
                 fs::write(
                     path.join(format!("{}_{}_{}.json", self.get_alias(), champ.id, pos)),
                     serde_json::to_string_pretty(&item_set).unwrap(),
@@ -85,7 +85,7 @@ pub trait DataSource {
                 .unwrap();
             }
             None => {
-                error!("Can't get data for {} at {}", champ.id, pos);
+                error!("{}: Can't get data for {} at {}", self.get_alias(), champ.id, pos);
             }
         }
     }

src/main.rs

@@ -1,7 +1,9 @@
 use indexmap::IndexMap;
+use lazy_static::lazy_static;
 #[cfg(target_os = "windows")]
 use log::debug;
 use log::{error, info, LevelFilter};
+use rayon::prelude::*;
 use serde_derive::Deserialize;
 use simple_logger::SimpleLogger;
 use std::env;
@@ -11,6 +13,7 @@ use std::io::Error;
 #[cfg(target_os = "windows")]
 use std::io::ErrorKind;
 use std::path::PathBuf;
+use std::time::Instant;
 use std::{fs, thread, time};
 use time::Duration;
 #[cfg(target_os = "windows")]
@@ -43,7 +46,6 @@ pub struct ChampInfo {
     key: String,
 }
 
-const USER_AGENT_KEY: &str = "User-Agent";
 const USER_AGENT_VALUE: &str =
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0";
 const DEFAULT_LOL_CHAMPS_DIR: &str = ".\\champs";
@@ -63,83 +65,65 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     for s in &args {
         if s.eq_ignore_ascii_case("-v") || s.eq_ignore_ascii_case("--verbose") {
             level = LevelFilter::Debug;
+            break;
         }
     }
     SimpleLogger::new()
         .with_level(level)
         .with_module_level("ureq", LevelFilter::Error)
+        .with_module_level("rustls", LevelFilter::Error)
         .init()?;
     info!("CGG Item Sets");
 
-    let lol_champs_dir = match lol_champ_dir() {
+    lazy_static! {
+        static ref LOL_CHAMPS_DIR: PathBuf = match lol_champ_dir() {
             Ok(x) => x,
             Err(_e) => PathBuf::from(DEFAULT_LOL_CHAMPS_DIR),
         };
-    info!(
-        "LoL Champs Folder: {}",
-        lol_champs_dir.to_str().unwrap_or(DEFAULT_LOL_CHAMPS_DIR)
-    );
-    let client = ureq::AgentBuilder::new()
+        static ref CLIENT: ureq::Agent = ureq::AgentBuilder::new()
+            .user_agent(USER_AGENT_VALUE)
             .timeout(Duration::from_secs(10))
             .build();
-
-    let realm: Realm = client
+        static ref REALM: Realm = CLIENT
             .get("https://ddragon.leagueoflegends.com/realms/euw.json")
-        .set(USER_AGENT_KEY, USER_AGENT_VALUE)
-        .call()?
-        .into_json()?;
-    info!("LoL version: {}", realm.v);
-
-    let champion: Champion = client
+            .call()
+            .unwrap()
+            .into_json()
+            .unwrap();
+        static ref CHAMPION: Champion = CLIENT
             .get(&format!(
                 "https://ddragon.leagueoflegends.com/cdn/{}/data/en_US/champion.json",
-            realm.v
+                REALM.v
             ))
-        .set(USER_AGENT_KEY, USER_AGENT_VALUE)
-        .call()?
-        .into_json()?;
-    info!("LoL numbers of champs: {}", champion.data.len());
-
-    let data_sources: [Box<dyn DataSource>; 3] = [
+            .call()
+            .unwrap()
+            .into_json()
+            .unwrap();
+        static ref DATA_SOURCES: Vec<Box<dyn DataSource + Sync + Send>> = vec![
             Box::new(PBDataSource),
             Box::new(CGGDataSource),
-        Box::new(KBDataSource::new(&client)),
+            Box::new(KBDataSource::new(&CLIENT)),
         ];
+    }
 
-    for data_source in data_sources.iter() {
-        let (champs, patch) = data_source.get_champs_with_positions_and_patch(&client);
+    info!("LoL Champs Folder: {}", LOL_CHAMPS_DIR.to_str().unwrap());
+    info!("LoL version: {}", REALM.v);
+    info!("LoL numbers of champs: {}", CHAMPION.data.len());
 
-        info!("{} version: {}", data_source.get_alias(), patch);
+    let mut threads = vec![];
+    for data_source in DATA_SOURCES.iter() {
+        threads.push(thread::spawn(move || {
+            let init = Instant::now();
+            execute_data_source(&data_source, &CLIENT, &CHAMPION, &LOL_CHAMPS_DIR);
             info!(
-            "{} numbers of champs: {}",
+                "{}: done in {} ms",
                 data_source.get_alias(),
-            champs.len()
+                init.elapsed().as_millis()
             );
-
-        for (id, positions) in &champs {
-            let mut champ_id = id.to_owned();
-
-            if !champion.data.contains_key(&champ_id) {
-                if let Some(c_id) = get_champ_from_key(&champion, &champ_id) {
-                    champ_id = c_id;
-                }
-            }
-
-            if let Some(champ) = champion.data.get(&champ_id) {
-                if positions.is_empty() {
-                    error!("{} missing positions", &champ_id);
-                } else {
-                    let path = lol_champs_dir.join(&champ_id).join("Recommended");
-                    fs::create_dir_all(&path)?;
-                    for pos in positions {
-                        data_source.write_item_set(&champ, &pos, &patch, &path, &client);
-                        thread::sleep(Duration::from_millis(data_source.get_timeout()));
-                    }
-                }
-            } else {
-                error!("{} not found in LoL champs", &champ_id);
-            }
+        }));
     }
+    for child in threads {
+        let _ = child.join();
     }
     Ok(())
 }
@@ -153,6 +137,81 @@ fn get_champ_from_key(champs: &Champion, key: &str) -> Option<String> {
     None
 }
 
+fn execute_data_source(
+    data_source: &Box<dyn DataSource + Sync + Send>,
+    client: &ureq::Agent,
+    champion: &Champion,
+    lol_champs_dir: &PathBuf,
+) {
+    let (champs, patch) = data_source.get_champs_with_positions_and_patch(&client);
+
+    info!("{} version: {}", data_source.get_alias(), patch);
+    info!(
+        "{} numbers of champs: {}",
+        data_source.get_alias(),
+        champs.len()
+    );
+
+    if data_source.get_timeout() == 0 {
+        champs.par_iter().for_each(|(id, positions)| {
+            get_and_write_item_set(
+                data_source,
+                client,
+                champion,
+                lol_champs_dir,
+                &patch,
+                id,
+                positions,
+            )
+        });
+    } else {
+        champs.iter().for_each(|(id, positions)| {
+            get_and_write_item_set(
+                data_source,
+                client,
+                champion,
+                lol_champs_dir,
+                &patch,
+                id,
+                positions,
+            )
+        });
+    };
+}
+
+fn get_and_write_item_set(
+    data_source: &Box<dyn DataSource + Sync + Send>,
+    client: &ureq::Agent,
+    champion: &Champion,
+    lol_champs_dir: &PathBuf,
+    patch: &String,
+    id: &String,
+    positions: &Vec<String>,
+) {
+    let mut champ_id: String = id.to_owned();
+
+    if id.parse::<u32>().is_ok() {
+        if let Some(c_id) = get_champ_from_key(&champion, &champ_id) {
+            champ_id = c_id;
+        }
+    }
+
+    if let Some(champ) = champion.data.get(&champ_id) {
+        if positions.is_empty() {
+            error!("{}: {} empty positions", data_source.get_alias(), &champ_id);
+        } else {
+            let path = lol_champs_dir.join(&champ_id).join("Recommended");
+            fs::create_dir_all(&path).unwrap();
+            positions.iter().for_each(|pos| {
+                data_source.write_item_set(&champ, &pos, &patch, &path, &client);
+                thread::sleep(Duration::from_millis(data_source.get_timeout()));
+            });
+        }
+    } else {
+        error!("{} not found in LoL champs", &champ_id);
+    }
+}
+
 #[cfg(target_os = "windows")]
 fn lol_champ_dir() -> Result<PathBuf, Error> {
     let hklm = RegKey::predef(winreg::enums::HKEY_LOCAL_MACHINE);
@@ -194,7 +253,7 @@ fn lol_champ_dir() -> Result<PathBuf, Error> {
     } else {
         return Err(Error::from(ErrorKind::NotFound));
     };
-    Ok(PathBuf::from(path).join("Config").join("Champions"))
+    Ok(path.join("Config").join("Champions"))
 }
 
 #[cfg(not(target_os = "windows"))]