refactor: enhance data handling in data sources
All checks were successful
ci/woodpecker/push/linux Pipeline was successful
ci/woodpecker/push/test Pipeline was successful
ci/woodpecker/push/mingw Pipeline was successful

This commit is contained in:
nyyu 2025-06-29 09:29:42 +02:00
parent c11c9620be
commit cabde1d433
5 changed files with 74 additions and 86 deletions

View file

@ -6,16 +6,16 @@ version = "1.0.0"
include = ["src/**/*"] include = ["src/**/*"]
[dependencies] [dependencies]
chrono = {version = "0.4.41", features = ["std"], default-features = false} chrono = { version = "0.4.41", features = ["std"], default-features = false }
indexmap = {version = "2.2", features = ["serde", "rayon"]} indexmap = { version = "2.2", features = ["serde", "rayon"] }
log = "0.4" log = "0.4"
logsy = "1.0.1" logsy = "1.0.1"
rayon = "1.10" rayon = "1.10"
regex = {version = "1.11.1", features = ["std"], default-features = false} regex = { version = "1.11.1", features = ["std"], default-features = false }
serde = "1.0" serde = "1.0"
serde_derive = "1.0" serde_derive = "1.0"
serde_json = {version = "1.0", features = ["preserve_order"]} serde_json = { version = "1.0", features = ["preserve_order"] }
ureq = {version = "3.0", features = ["json"]} ureq = { version = "3.0", features = ["json"] }
[target.'cfg(windows)'.dependencies] [target.'cfg(windows)'.dependencies]
winreg = { version = "0.55" } winreg = { version = "0.55" }

View file

@ -49,13 +49,21 @@ pub struct Stat {
pub patch: String, pub patch: String,
} }
pub struct Data {
pub position: String,
pub items: Vec<Value>,
pub stat: Stat,
}
/// Trait for a data source that can provide champion item set data. /// Trait for a data source that can provide champion item set data.
pub trait DataSource { pub trait DataSource {
/// Returns the alias of the data source. /// Returns the alias of the data source.
fn get_alias(&self) -> &str; fn get_alias(&self) -> &str;
/// Returns the timeout for the data source. /// Returns the timeout for the data source.
fn get_timeout(&self) -> u64; fn get_timeout(&self) -> u64 {
300
}
/// Returns a map of champion IDs to their possible positions. /// Returns a map of champion IDs to their possible positions.
fn get_champs_with_positions(&self, champion: &Champion) -> IndexMap<u32, Vec<String>>; fn get_champs_with_positions(&self, champion: &Champion) -> IndexMap<u32, Vec<String>>;
@ -68,12 +76,29 @@ pub trait DataSource {
}) })
} }
/// Logs missing roles for which data could not be retrieved.
fn log_missing_roles(&self, champ: &ChampInfo, positions: &[String], data: &[Data]) {
let missing_roles: Vec<_> = positions
.iter()
.filter(|pos| !data.iter().any(|build| &build.position == *pos))
.cloned()
.collect();
if !missing_roles.is_empty() {
error!(
"{}: Can't get data for {} at {}",
self.get_alias(),
champ.id,
missing_roles.join(", ")
);
}
}
/// Returns champion data with win percentage for the given positions. /// Returns champion data with win percentage for the given positions.
fn get_champ_data_with_win_pourcentage( fn get_champ_data_with_win_pourcentage(
&self, &self,
champ: &ChampInfo, champ: &ChampInfo,
positions: &[String], positions: &[String],
) -> Vec<(String, Vec<Value>, Stat)>; ) -> Vec<Data>;
/// Writes item sets for the given champion and positions to the specified path. /// Writes item sets for the given champion and positions to the specified path.
fn write_item_set( fn write_item_set(
@ -90,59 +115,47 @@ pub trait DataSource {
); );
let data = self.get_champ_data_with_win_pourcentage(champ, positions); let data = self.get_champ_data_with_win_pourcentage(champ, positions);
let missing_roles: Vec<_> = positions self.log_missing_roles(champ, positions, &data);
.iter()
.filter(|pos| !data.iter().any(|build| &build.0 == *pos))
.cloned()
.collect();
if !missing_roles.is_empty() {
error!(
"{}: Can't get data for {} at {}",
self.get_alias(),
champ.id,
missing_roles.join(", ")
);
}
for build in data { for build in data {
let item_set = ItemSet { let item_set = ItemSet {
title: format!( title: format!(
"{} {} {} - {:.2}% wins - {} games - {:.2} kda", "{} {} {} - {:.2}% wins - {} games - {:.2} kda",
self.get_alias(), self.get_alias(),
build.0, build.position,
build.2.patch, build.stat.patch,
build.2.win_rate, build.stat.win_rate,
build.2.games, build.stat.games,
build.2.kda build.stat.kda
), ),
type_: "custom".to_string(), type_: "custom".to_owned(),
map: "any".to_string(), map: "any".to_owned(),
mode: "any".to_string(), mode: "any".to_owned(),
priority: false, priority: false,
sortrank: 0, sortrank: 0,
blocks: build.1, blocks: build.items,
}; };
info!( info!(
"{}: Writing item set for {} at {}", "{}: Writing item set for {} at {}",
self.get_alias(), self.get_alias(),
champ.name, champ.name,
build.0 build.position
); );
let json_string = serde_json::to_string_pretty(&item_set) let json_string = serde_json::to_string_pretty(&item_set)
.map_err(|e| format!("Failed to serialize item set: {}", e))?; .map_err(|e| format!("Failed to serialize item set: {e}"))?;
fs::write( fs::write(
path.join(format!( path.join(format!(
"{}_{}_{}.json", "{}_{}_{}.json",
self.get_alias(), self.get_alias(),
champ.id, champ.id,
build.0 build.position
)), )),
json_string, json_string,
) )
.map_err(|e| format!("Failed to write item set file: {}", e))?; .map_err(|e| format!("Failed to write item set file: {e}"))?;
} }
Ok(()) Ok(())
} }

View file

@ -1,10 +1,9 @@
use crate::ChampInfo; use crate::ChampInfo;
use crate::Champion as ChampionLoL; use crate::Champion as ChampionLoL;
use crate::data_source::{DataSource, Item, Stat}; use crate::data_source::{Data, DataSource, Item, Stat};
use indexmap::IndexMap; use indexmap::IndexMap;
use log::error; use log::error;
use serde_derive::Deserialize; use serde_derive::Deserialize;
use serde_json::Value;
pub struct KBDataSource { pub struct KBDataSource {
client: ureq::Agent, client: ureq::Agent,
@ -140,7 +139,7 @@ impl KBDataSource {
let start = bundle.find(auth_marker)? + auth_marker.len(); let start = bundle.find(auth_marker)? + auth_marker.len();
let after_marker = bundle[start..].find('"')? + start + 1; let after_marker = bundle[start..].find('"')? + start + 1;
let end = bundle[after_marker..].find('"')? + after_marker; let end = bundle[after_marker..].find('"')? + after_marker;
Some(bundle[after_marker..end].to_string()) Some(bundle[after_marker..end].to_owned())
} }
fn get_champion_response(&self) -> Option<ChampionResponse> { fn get_champion_response(&self) -> Option<ChampionResponse> {
@ -178,7 +177,7 @@ impl KBDataSource {
} }
} }
fn get_build(&self, build: &KBBuild) -> (String, Vec<Value>, Stat) { fn get_build(&self, build: &KBBuild) -> Data {
let mut starting_items: Vec<Item> = vec![]; let mut starting_items: Vec<Item> = vec![];
let mut blocks = vec![]; let mut blocks = vec![];
for i in 0..6 { for i in 0..6 {
@ -229,16 +228,16 @@ impl KBDataSource {
), ),
)); ));
( Data {
build.position.position_name.to_uppercase(), position: build.position.position_name.to_uppercase(),
blocks, items: blocks,
Stat { stat: Stat {
win_rate: (build.wins as f32 / build.games as f32) * 100f32, win_rate: (build.wins as f32 / build.games as f32) * 100f32,
games: build.games, games: build.games,
kda: build.kda, kda: build.kda,
patch: build.patch.patch_version.to_owned(), patch: build.patch.patch_version.to_owned(),
}, },
) }
} }
} }
@ -247,10 +246,6 @@ impl DataSource for KBDataSource {
"KB" "KB"
} }
fn get_timeout(&self) -> u64 {
300
}
fn get_champs_with_positions(&self, _champion: &ChampionLoL) -> IndexMap<u32, Vec<String>> { fn get_champs_with_positions(&self, _champion: &ChampionLoL) -> IndexMap<u32, Vec<String>> {
let mut champions = IndexMap::new(); let mut champions = IndexMap::new();
let data: ChampionResponse = match self.get_champion_response() { let data: ChampionResponse = match self.get_champion_response() {
@ -269,7 +264,7 @@ impl DataSource for KBDataSource {
&self, &self,
champ: &ChampInfo, champ: &ChampInfo,
position: &[String], position: &[String],
) -> Vec<(String, Vec<Value>, Stat)> { ) -> Vec<Data> {
let mut champ_data = vec![]; let mut champ_data = vec![];
if let Some(token) = &self.token { if let Some(token) = &self.token {
let url = format!( let url = format!(
@ -286,12 +281,12 @@ impl DataSource for KBDataSource {
Ok(mut resp) => match resp.body_mut().read_json() { Ok(mut resp) => match resp.body_mut().read_json() {
Ok(val) => val, Ok(val) => val,
Err(x) => { Err(x) => {
error!("Cant json: {}", x); error!("Cant json: {x}");
return vec![]; return vec![];
} }
}, },
Err(x) => { Err(x) => {
error!("Call failed for URL: {}, error: {}", url, x); error!("Call failed for URL: {url}, error: {x}");
return vec![]; return vec![];
} }
}; };
@ -343,9 +338,9 @@ mod tests {
key: 1, key: 1,
}; };
let result = let result =
DATASOURCE.get_champ_data_with_win_pourcentage(&champ, &vec!["MID".to_string()]); DATASOURCE.get_champ_data_with_win_pourcentage(&champ, &vec!["MID".to_owned()]);
assert!(!result.is_empty()); assert!(!result.is_empty());
assert!(!result[0].1.is_empty()); assert!(!result[0].items.is_empty());
assert!(result[0].2.win_rate > 0.); assert!(result[0].stat.win_rate > 0.);
} }
} }

View file

@ -234,29 +234,20 @@ fn get_browser_user_agent() -> String {
fn lol_champ_dir() -> Result<PathBuf, Error> { fn lol_champ_dir() -> Result<PathBuf, Error> {
let hklm = RegKey::predef(winreg::enums::HKEY_LOCAL_MACHINE); let hklm = RegKey::predef(winreg::enums::HKEY_LOCAL_MACHINE);
let path = if let Ok(node) = hklm.open_subkey(REG_KEY_LOL_RADS) { let path = if let Ok(node) = hklm.open_subkey(REG_KEY_LOL_RADS) {
debug!( debug!("Use registry key {REG_KEY_LOL_RADS} for relative champ directory");
"Use registry key {} for relative champ directory",
REG_KEY_LOL_RADS
);
let val: String = node.get_value("LocalRootFolder")?; let val: String = node.get_value("LocalRootFolder")?;
match PathBuf::from(val).parent() { match PathBuf::from(val).parent() {
Some(parent) => parent.to_path_buf(), Some(parent) => parent.to_path_buf(),
None => return Err(Error::from(ErrorKind::NotFound)), None => return Err(Error::from(ErrorKind::NotFound)),
} }
} else if let Ok(node) = hklm.open_subkey(REG_KEY_LOL_INC) { } else if let Ok(node) = hklm.open_subkey(REG_KEY_LOL_INC) {
debug!( debug!("Use registry key {REG_KEY_LOL_INC} for relative champ directory");
"Use registry key {} for relative champ directory",
REG_KEY_LOL_INC
);
let val: String = node.get_value("Location")?; let val: String = node.get_value("Location")?;
PathBuf::from(val) PathBuf::from(val)
} else if let Ok(node) = } else if let Ok(node) =
find_subnode_from_path(hklm, REG_KEY_WIN_64_UNINSTALL, "League of Legends") find_subnode_from_path(hklm, REG_KEY_WIN_64_UNINSTALL, "League of Legends")
{ {
debug!( debug!("Use registry key {REG_KEY_WIN_64_UNINSTALL} for relative champ directory");
"Use registry key {} for relative champ directory",
REG_KEY_WIN_64_UNINSTALL
);
let val: String = node.get_value("InstallLocation")?; let val: String = node.get_value("InstallLocation")?;
PathBuf::from(val) PathBuf::from(val)
} else if let Ok(node) = find_subnode_from_path( } else if let Ok(node) = find_subnode_from_path(
@ -264,10 +255,7 @@ fn lol_champ_dir() -> Result<PathBuf, Error> {
REG_KEY_WIN_UNINSTALL, REG_KEY_WIN_UNINSTALL,
"Riot Game league_of_legends.live", "Riot Game league_of_legends.live",
) { ) {
debug!( debug!("Use registry key {REG_KEY_WIN_UNINSTALL} for relative champ directory");
"Use registry key {} for relative champ directory",
REG_KEY_WIN_UNINSTALL
);
let val: String = node.get_value("InstallLocation")?; let val: String = node.get_value("InstallLocation")?;
PathBuf::from(val) PathBuf::from(val)
} else { } else {

View file

@ -2,12 +2,11 @@ use indexmap::IndexMap;
use log::{error, warn}; use log::{error, warn};
use regex::Regex; use regex::Regex;
use serde_derive::Deserialize; use serde_derive::Deserialize;
use serde_json::Value;
use std::sync::LazyLock; use std::sync::LazyLock;
use crate::ChampInfo; use crate::ChampInfo;
use crate::Champion; use crate::Champion;
use crate::data_source::{DataSource, Stat}; use crate::data_source::{Data, DataSource, Stat};
pub struct MSDataSource { pub struct MSDataSource {
client: ureq::Agent, client: ureq::Agent,
@ -76,13 +75,10 @@ fn extract_items_from_section(page: &str, section_title: &str) -> Vec<String> {
.map(|cap| cap[1].to_owned()) .map(|cap| cap[1].to_owned())
.collect(); .collect();
} else { } else {
warn!( warn!("Failed to find matching </div> for section '{section_title}'");
"Failed to find matching </div> for section '{}'",
section_title
);
} }
} }
Vec::new() vec![]
} }
fn extract_skill_order_from_table(page: &str) -> String { fn extract_skill_order_from_table(page: &str) -> String {
@ -145,10 +141,6 @@ impl DataSource for MSDataSource {
"MS" "MS"
} }
fn get_timeout(&self) -> u64 {
300
}
fn get_champs_with_positions(&self, champion: &Champion) -> IndexMap<u32, Vec<String>> { fn get_champs_with_positions(&self, champion: &Champion) -> IndexMap<u32, Vec<String>> {
let mut champs = IndexMap::new(); let mut champs = IndexMap::new();
@ -160,7 +152,7 @@ impl DataSource for MSDataSource {
{ {
Ok(champs) => champs, Ok(champs) => champs,
Err(e) => { Err(e) => {
error!("Failed to fetch champions from MetaSRC: {}", e); error!("Failed to fetch champions from MetaSRC: {e}");
return champs; return champs;
} }
}; };
@ -189,7 +181,7 @@ impl DataSource for MSDataSource {
&self, &self,
champ: &ChampInfo, champ: &ChampInfo,
positions: &[String], positions: &[String],
) -> Vec<(String, Vec<Value>, Stat)> { ) -> Vec<Data> {
let mut builds = vec![]; let mut builds = vec![];
let rep = self let rep = self
@ -236,9 +228,9 @@ impl DataSource for MSDataSource {
let items = extract_items_from_section(&page, "Item Purchase Order"); let items = extract_items_from_section(&page, "Item Purchase Order");
let starting_items = extract_items_from_section(&page, "Starting Items"); let starting_items = extract_items_from_section(&page, "Starting Items");
builds.push(( builds.push(Data {
positions[0].to_owned(), position: positions[0].to_owned(),
vec![ items: vec![
self.make_item_set( self.make_item_set(
starting_items, starting_items,
format!( format!(
@ -248,13 +240,13 @@ impl DataSource for MSDataSource {
), ),
self.make_item_set(items, "Item Purchase Order".to_owned()), self.make_item_set(items, "Item Purchase Order".to_owned()),
], ],
Stat { stat: Stat {
win_rate, win_rate,
games, games,
kda, kda,
patch, patch,
}, },
)); });
} else { } else {
warn!( warn!(
"Failed to fetch build page for champ {} at position {}", "Failed to fetch build page for champ {} at position {}",