nyyu/CGGItemSets
1
1
Fork 1
Code Issues 1 Pull Requests Releases Wiki Activity

more threads

Browse Source
This commit is contained in:
nyyu 2021-03-15 22:09:30 +01:00 committed by Gitea
parent 5117bbe659
commit 6fcf6d4105
3 changed files with 159 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
Cargo.lock generated
View File

@ -48,6 +48,7 @@ dependencies = [
"indexmap",
"lazy_static",
"log",
"rayon",
"serde",
"serde_derive",
"serde_json",
@ -86,6 +87,57 @@ dependencies = [
"winapi",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
dependencies = [
"cfg-if",
"crossbeam-utils",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
dependencies = [
"autocfg",
"cfg-if",
"lazy_static",
]
[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "encoding_rs"
version = "0.8.28"
@ -139,6 +191,7 @@ checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3"
dependencies = [
"autocfg",
"hashbrown",
"rayon",
"serde",
]
@ -184,6 +237,15 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
[[package]]
name = "memoffset"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
dependencies = [
"autocfg",
]
[[package]]
name = "num-integer"
version = "0.1.44"
@ -203,6 +265,16 @@ dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "once_cell"
version = "1.7.2"
@ -233,6 +305,31 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "rayon"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"lazy_static",
"num_cpus",
]
[[package]]
name = "ring"
version = "0.16.20"
@ -267,6 +364,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sct"
version = "0.6.0"

3
Cargo.toml
View File

@ -5,9 +5,10 @@ name = "cggitem_sets"
version = "0.1.0"
[dependencies]
indexmap = {version = "1.6", features = ["serde-1"]}
indexmap = {version = "1.6", features = ["serde-1", "rayon"]}
lazy_static = "1.4"
log = "0.4"
rayon = "1.5"
serde = "1.0"
serde_derive = "1.0"
serde_json = {version = "1.0", features = ["preserve_order"]}

73
src/main.rs
View File

@ -3,6 +3,7 @@ use lazy_static::lazy_static;
#[cfg(target_os = "windows")]
use log::debug;
use log::{error, info, LevelFilter};
use rayon::prelude::*;
use serde_derive::Deserialize;
use simple_logger::SimpleLogger;
use std::env;
@ -98,7 +99,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
.unwrap()
.into_json()
.unwrap();
static ref DATA_SOURCES: [Box<dyn DataSource + Sync + Send>; 3] = [
static ref DATA_SOURCES: Vec<Box<dyn DataSource + Sync + Send>> = vec![
Box::new(PBDataSource),
Box::new(CGGDataSource),
Box::new(KBDataSource::new(&CLIENT)),
@ -151,29 +152,63 @@ fn execute_data_source(
champs.len()
);
for (id, positions) in &champs {
let mut champ_id: String = id.to_owned();
if data_source.get_timeout() == 0 {
champs.par_iter().for_each(|(id, positions)| {
get_and_write_item_set(
data_source,
client,
champion,
lol_champs_dir,
&patch,
id,
positions,
)
});
} else {
champs.iter().for_each(|(id, positions)| {
get_and_write_item_set(
data_source,
client,
champion,
lol_champs_dir,
&patch,
id,
positions,
)
});
};
}
if !champion.data.contains_key(&champ_id) {
if let Some(c_id) = get_champ_from_key(&champion, &champ_id) {
champ_id = c_id;
}
fn get_and_write_item_set(
data_source: &Box<dyn DataSource + Sync + Send>,
client: &ureq::Agent,
champion: &Champion,
lol_champs_dir: &PathBuf,
patch: &String,
id: &String,
positions: &Vec<String>,
) {
let mut champ_id: String = id.to_owned();
if id.parse::<u32>().is_ok() {
if let Some(c_id) = get_champ_from_key(&champion, &champ_id) {
champ_id = c_id;
}
}
if let Some(champ) = champion.data.get(&champ_id) {
if positions.is_empty() {
error!("{}: {} empty positions", data_source.get_alias(), &champ_id);
} else {
let path = lol_champs_dir.join(&champ_id).join("Recommended");
fs::create_dir_all(&path).unwrap();
for pos in positions {
data_source.write_item_set(&champ, &pos, &patch, &path, &client);
thread::sleep(Duration::from_millis(data_source.get_timeout()));
}
}
if let Some(champ) = champion.data.get(&champ_id) {
if positions.is_empty() {
error!("{}: {} empty positions", data_source.get_alias(), &champ_id);
} else {
error!("{} not found in LoL champs", &champ_id);
let path = lol_champs_dir.join(&champ_id).join("Recommended");
fs::create_dir_all(&path).unwrap();
positions.iter().for_each(|pos| {
data_source.write_item_set(&champ, &pos, &patch, &path, &client);
thread::sleep(Duration::from_millis(data_source.get_timeout()));
});
}
} else {
error!("{} not found in LoL champs", &champ_id);
}
}