From 5b39644df0b33b17831cf74615b21cdfd4f661f6 Mon Sep 17 00:00:00 2001 From: nyyu Date: Mon, 15 Mar 2021 18:59:49 +0100 Subject: [PATCH 1/3] some threads --- src/main.rs | 153 +++++++++++++++++++++++++++++----------------------- 1 file changed, 86 insertions(+), 67 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1320710..ddfb5d9 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use indexmap::IndexMap; +use lazy_static::lazy_static; #[cfg(target_os = "windows")] use log::debug; use log::{error, info, LevelFilter}; @@ -71,75 +72,52 @@ fn main() -> Result<(), Box> { .init()?; info!("CGG Item Sets"); - let lol_champs_dir = match lol_champ_dir() { - Ok(x) => x, - Err(_e) => PathBuf::from(DEFAULT_LOL_CHAMPS_DIR), - }; + lazy_static! { + static ref LOL_CHAMPS_DIR: PathBuf = match lol_champ_dir() { + Ok(x) => x, + Err(_e) => PathBuf::from(DEFAULT_LOL_CHAMPS_DIR), + }; + static ref CLIENT: ureq::Agent = ureq::AgentBuilder::new() + .timeout(Duration::from_secs(10)) + .build(); + static ref REALM: Realm = CLIENT + .get("https://ddragon.leagueoflegends.com/realms/euw.json") + .set(USER_AGENT_KEY, USER_AGENT_VALUE) + .call() + .unwrap() + .into_json() + .unwrap(); + static ref CHAMPION: Champion = CLIENT + .get(&format!( + "https://ddragon.leagueoflegends.com/cdn/{}/data/en_US/champion.json", + REALM.v + )) + .set(USER_AGENT_KEY, USER_AGENT_VALUE) + .call() + .unwrap() + .into_json() + .unwrap(); + static ref DATA_SOURCES: [Box; 3] = [ + Box::new(PBDataSource), + Box::new(CGGDataSource), + Box::new(KBDataSource::new(&CLIENT)), + ]; + } + info!( - "LoL Champs Folder: {}", - lol_champs_dir.to_str().unwrap_or(DEFAULT_LOL_CHAMPS_DIR) + "LoL Champs Folder: {}", LOL_CHAMPS_DIR.to_str().unwrap() ); - let client = ureq::AgentBuilder::new() - .timeout(Duration::from_secs(10)) - .build(); + info!("LoL version: {}", REALM.v); + info!("LoL numbers of champs: {}", CHAMPION.data.len()); - let realm: Realm = client - .get("https://ddragon.leagueoflegends.com/realms/euw.json") - .set(USER_AGENT_KEY, USER_AGENT_VALUE) - .call()? - .into_json()?; - info!("LoL version: {}", realm.v); - - let champion: Champion = client - .get(&format!( - "https://ddragon.leagueoflegends.com/cdn/{}/data/en_US/champion.json", - realm.v - )) - .set(USER_AGENT_KEY, USER_AGENT_VALUE) - .call()? - .into_json()?; - info!("LoL numbers of champs: {}", champion.data.len()); - - let data_sources: [Box; 3] = [ - Box::new(PBDataSource), - Box::new(CGGDataSource), - Box::new(KBDataSource::new(&client)), - ]; - - for data_source in data_sources.iter() { - let (champs, patch) = data_source.get_champs_with_positions_and_patch(&client); - - info!("{} version: {}", data_source.get_alias(), patch); - info!( - "{} numbers of champs: {}", - data_source.get_alias(), - champs.len() - ); - - for (id, positions) in &champs { - let mut champ_id = id.to_owned(); - - if !champion.data.contains_key(&champ_id) { - if let Some(c_id) = get_champ_from_key(&champion, &champ_id) { - champ_id = c_id; - } - } - - if let Some(champ) = champion.data.get(&champ_id) { - if positions.is_empty() { - error!("{} missing positions", &champ_id); - } else { - let path = lol_champs_dir.join(&champ_id).join("Recommended"); - fs::create_dir_all(&path)?; - for pos in positions { - data_source.write_item_set(&champ, &pos, &patch, &path, &client); - thread::sleep(Duration::from_millis(data_source.get_timeout())); - } - } - } else { - error!("{} not found in LoL champs", &champ_id); - } - } + let mut threads = vec![]; + for data_source in DATA_SOURCES.iter() { + threads.push(thread::spawn(move || { + execute_data_source(&data_source, &CLIENT, &CHAMPION, &LOL_CHAMPS_DIR) + })); + } + for child in threads { + let _ = child.join(); } Ok(()) } @@ -153,6 +131,47 @@ fn get_champ_from_key(champs: &Champion, key: &str) -> Option { None } +fn execute_data_source( + data_source: &Box, + client: &ureq::Agent, + champion: &Champion, + lol_champs_dir: &PathBuf, +) { + let (champs, patch) = data_source.get_champs_with_positions_and_patch(&client); + + info!("{} version: {}", data_source.get_alias(), patch); + info!( + "{} numbers of champs: {}", + data_source.get_alias(), + champs.len() + ); + + for (id, positions) in &champs { + let mut champ_id: String = id.to_owned(); + + if !champion.data.contains_key(&champ_id) { + if let Some(c_id) = get_champ_from_key(&champion, &champ_id) { + champ_id = c_id; + } + } + + if let Some(champ) = champion.data.get(&champ_id) { + if positions.is_empty() { + error!("{} missing positions", &champ_id); + } else { + let path = lol_champs_dir.join(&champ_id).join("Recommended"); + fs::create_dir_all(&path).unwrap(); + for pos in positions { + data_source.write_item_set(&champ, &pos, &patch, &path, &client); + thread::sleep(Duration::from_millis(data_source.get_timeout())); + } + } + } else { + error!("{} not found in LoL champs", &champ_id); + } + } +} + #[cfg(target_os = "windows")] fn lol_champ_dir() -> Result { let hklm = RegKey::predef(winreg::enums::HKEY_LOCAL_MACHINE); @@ -194,7 +213,7 @@ fn lol_champ_dir() -> Result { } else { return Err(Error::from(ErrorKind::NotFound)); }; - Ok(PathBuf::from(path).join("Config").join("Champions")) + Ok(path.join("Config").join("Champions")) } #[cfg(not(target_os = "windows"))] From 5117bbe6593042e6cf8e85feb8a37db9599948c3 Mon Sep 17 00:00:00 2001 From: nyyu Date: Mon, 15 Mar 2021 21:15:03 +0100 Subject: [PATCH 2/3] Improve log --- src/data_source.rs | 6 +++--- src/main.rs | 21 +++++++++++++-------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/data_source.rs b/src/data_source.rs index 924c936..f9fc719 100644 --- a/src/data_source.rs +++ b/src/data_source.rs @@ -62,7 +62,7 @@ pub trait DataSource { path: &PathBuf, client: &ureq::Agent, ) { - info!("Retrieving data for {} at {}", champ.name, pos); + info!("{}: Retrieving data for {} at {}", self.get_alias(), champ.name, pos); let data = self.get_champ_data_with_win_pourcentage(champ, pos, client); match data { @@ -77,7 +77,7 @@ pub trait DataSource { blocks: data.0, }; - info!("Writing item set for {} at {}", champ.name, pos); + info!("{}: Writing item set for {} at {}", self.get_alias(), champ.name, pos); fs::write( path.join(format!("{}_{}_{}.json", self.get_alias(), champ.id, pos)), serde_json::to_string_pretty(&item_set).unwrap(), @@ -85,7 +85,7 @@ pub trait DataSource { .unwrap(); } None => { - error!("Can't get data for {} at {}", champ.id, pos); + error!("{}: Can't get data for {} at {}", self.get_alias(), champ.id, pos); } } } diff --git a/src/main.rs b/src/main.rs index ddfb5d9..f9b164c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,6 +12,7 @@ use std::io::Error; #[cfg(target_os = "windows")] use std::io::ErrorKind; use std::path::PathBuf; +use std::time::Instant; use std::{fs, thread, time}; use time::Duration; #[cfg(target_os = "windows")] @@ -44,7 +45,6 @@ pub struct ChampInfo { key: String, } -const USER_AGENT_KEY: &str = "User-Agent"; const USER_AGENT_VALUE: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0"; const DEFAULT_LOL_CHAMPS_DIR: &str = ".\\champs"; @@ -64,11 +64,13 @@ fn main() -> Result<(), Box> { for s in &args { if s.eq_ignore_ascii_case("-v") || s.eq_ignore_ascii_case("--verbose") { level = LevelFilter::Debug; + break; } } SimpleLogger::new() .with_level(level) .with_module_level("ureq", LevelFilter::Error) + .with_module_level("rustls", LevelFilter::Error) .init()?; info!("CGG Item Sets"); @@ -78,11 +80,11 @@ fn main() -> Result<(), Box> { Err(_e) => PathBuf::from(DEFAULT_LOL_CHAMPS_DIR), }; static ref CLIENT: ureq::Agent = ureq::AgentBuilder::new() + .user_agent(USER_AGENT_VALUE) .timeout(Duration::from_secs(10)) .build(); static ref REALM: Realm = CLIENT .get("https://ddragon.leagueoflegends.com/realms/euw.json") - .set(USER_AGENT_KEY, USER_AGENT_VALUE) .call() .unwrap() .into_json() @@ -92,7 +94,6 @@ fn main() -> Result<(), Box> { "https://ddragon.leagueoflegends.com/cdn/{}/data/en_US/champion.json", REALM.v )) - .set(USER_AGENT_KEY, USER_AGENT_VALUE) .call() .unwrap() .into_json() @@ -104,16 +105,20 @@ fn main() -> Result<(), Box> { ]; } - info!( - "LoL Champs Folder: {}", LOL_CHAMPS_DIR.to_str().unwrap() - ); + info!("LoL Champs Folder: {}", LOL_CHAMPS_DIR.to_str().unwrap()); info!("LoL version: {}", REALM.v); info!("LoL numbers of champs: {}", CHAMPION.data.len()); let mut threads = vec![]; for data_source in DATA_SOURCES.iter() { threads.push(thread::spawn(move || { - execute_data_source(&data_source, &CLIENT, &CHAMPION, &LOL_CHAMPS_DIR) + let init = Instant::now(); + execute_data_source(&data_source, &CLIENT, &CHAMPION, &LOL_CHAMPS_DIR); + info!( + "{}: done in {} ms", + data_source.get_alias(), + init.elapsed().as_millis() + ); })); } for child in threads { @@ -157,7 +162,7 @@ fn execute_data_source( if let Some(champ) = champion.data.get(&champ_id) { if positions.is_empty() { - error!("{} missing positions", &champ_id); + error!("{}: {} empty positions", data_source.get_alias(), &champ_id); } else { let path = lol_champs_dir.join(&champ_id).join("Recommended"); fs::create_dir_all(&path).unwrap(); From 6fcf6d4105ce66a1d5630e03bebfc989e532f979 Mon Sep 17 00:00:00 2001 From: nyyu Date: Mon, 15 Mar 2021 22:09:30 +0100 Subject: [PATCH 3/3] more threads --- Cargo.lock | 103 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 3 +- src/main.rs | 73 +++++++++++++++++++++++++++---------- 3 files changed, 159 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f8697d5..c3ca387 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -48,6 +48,7 @@ dependencies = [ "indexmap", "lazy_static", "log", + "rayon", "serde", "serde_derive", "serde_json", @@ -86,6 +87,57 @@ dependencies = [ "winapi", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" +dependencies = [ + "cfg-if", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49" +dependencies = [ + "autocfg", + "cfg-if", + "lazy_static", +] + +[[package]] +name = "either" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" + [[package]] name = "encoding_rs" version = "0.8.28" @@ -139,6 +191,7 @@ checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", "hashbrown", + "rayon", "serde", ] @@ -184,6 +237,15 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + [[package]] name = "num-integer" version = "0.1.44" @@ -203,6 +265,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "once_cell" version = "1.7.2" @@ -233,6 +305,31 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "ring" version = "0.16.20" @@ -267,6 +364,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "sct" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index 66a176e..a21c6a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,9 +5,10 @@ name = "cggitem_sets" version = "0.1.0" [dependencies] -indexmap = {version = "1.6", features = ["serde-1"]} +indexmap = {version = "1.6", features = ["serde-1", "rayon"]} lazy_static = "1.4" log = "0.4" +rayon = "1.5" serde = "1.0" serde_derive = "1.0" serde_json = {version = "1.0", features = ["preserve_order"]} diff --git a/src/main.rs b/src/main.rs index f9b164c..69923f1 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ use lazy_static::lazy_static; #[cfg(target_os = "windows")] use log::debug; use log::{error, info, LevelFilter}; +use rayon::prelude::*; use serde_derive::Deserialize; use simple_logger::SimpleLogger; use std::env; @@ -98,7 +99,7 @@ fn main() -> Result<(), Box> { .unwrap() .into_json() .unwrap(); - static ref DATA_SOURCES: [Box; 3] = [ + static ref DATA_SOURCES: Vec> = vec![ Box::new(PBDataSource), Box::new(CGGDataSource), Box::new(KBDataSource::new(&CLIENT)), @@ -151,29 +152,63 @@ fn execute_data_source( champs.len() ); - for (id, positions) in &champs { - let mut champ_id: String = id.to_owned(); + if data_source.get_timeout() == 0 { + champs.par_iter().for_each(|(id, positions)| { + get_and_write_item_set( + data_source, + client, + champion, + lol_champs_dir, + &patch, + id, + positions, + ) + }); + } else { + champs.iter().for_each(|(id, positions)| { + get_and_write_item_set( + data_source, + client, + champion, + lol_champs_dir, + &patch, + id, + positions, + ) + }); + }; +} - if !champion.data.contains_key(&champ_id) { - if let Some(c_id) = get_champ_from_key(&champion, &champ_id) { - champ_id = c_id; - } +fn get_and_write_item_set( + data_source: &Box, + client: &ureq::Agent, + champion: &Champion, + lol_champs_dir: &PathBuf, + patch: &String, + id: &String, + positions: &Vec, +) { + let mut champ_id: String = id.to_owned(); + + if id.parse::().is_ok() { + if let Some(c_id) = get_champ_from_key(&champion, &champ_id) { + champ_id = c_id; } + } - if let Some(champ) = champion.data.get(&champ_id) { - if positions.is_empty() { - error!("{}: {} empty positions", data_source.get_alias(), &champ_id); - } else { - let path = lol_champs_dir.join(&champ_id).join("Recommended"); - fs::create_dir_all(&path).unwrap(); - for pos in positions { - data_source.write_item_set(&champ, &pos, &patch, &path, &client); - thread::sleep(Duration::from_millis(data_source.get_timeout())); - } - } + if let Some(champ) = champion.data.get(&champ_id) { + if positions.is_empty() { + error!("{}: {} empty positions", data_source.get_alias(), &champ_id); } else { - error!("{} not found in LoL champs", &champ_id); + let path = lol_champs_dir.join(&champ_id).join("Recommended"); + fs::create_dir_all(&path).unwrap(); + positions.iter().for_each(|pos| { + data_source.write_item_set(&champ, &pos, &patch, &path, &client); + thread::sleep(Duration::from_millis(data_source.get_timeout())); + }); } + } else { + error!("{} not found in LoL champs", &champ_id); } }