Merge pull request 'feat: migrate from reqwest to ureq' (#6) from slany/CGGItemSets:feat/ureq into master
All checks were successful
continuous-integration/drone/push Build is passing

Reviewed-on: #6
This commit is contained in:
nyyu 2021-03-10 12:48:29 +01:00
commit fa179fe465
7 changed files with 168 additions and 811 deletions

869
Cargo.lock generated

File diff suppressed because it is too large Load diff

View file

@ -10,14 +10,14 @@ simple_logger = "1.11"
serde = "1.0" serde = "1.0"
serde_derive = "1.0" serde_derive = "1.0"
serde_json = { version = "1.0", features = ["preserve_order"] } serde_json = { version = "1.0", features = ["preserve_order"] }
reqwest = { version = "0.10", features = ["blocking", "json"] }
select = "0.5" select = "0.5"
regex = "1.4" regex = "1.4"
lazy_static = "1.4" lazy_static = "1.4"
indexmap = { version = "1.6", features = ["serde-1"] } indexmap = { version = "1.6", features = ["serde-1"] }
ureq = { version = "2.0", features = ["json", "charset"] }
[target.'cfg(windows)'.dependencies] [target.'cfg(windows)'.dependencies]
winreg = "0.7" winreg = "0.8"
[profile.release] [profile.release]
opt-level = 'z' opt-level = 'z'

View file

@ -65,13 +65,13 @@ impl DataSource for CGGDataSource {
fn get_champs_with_positions_and_patch( fn get_champs_with_positions_and_patch(
&self, &self,
client: &reqwest::blocking::Client, client: &ureq::Agent,
) -> (IndexMap<String, Vec<String>>, String) { ) -> (IndexMap<String, Vec<String>>, String) {
let page = client let page = client
.get("https://champion.gg") .get("https://champion.gg")
.send() .call()
.unwrap() .unwrap()
.text() .into_string()
.unwrap(); .unwrap();
let document = Document::from(&*page); let document = Document::from(&*page);
@ -118,21 +118,23 @@ impl DataSource for CGGDataSource {
&self, &self,
id: &str, id: &str,
position: &str, position: &str,
client: &reqwest::blocking::Client, client: &ureq::Agent,
) -> Option<(Vec<Value>, f64)> { ) -> Option<(Vec<Value>, f64)> {
let req = client let req = client
.get(&format!( .get(&format!(
"https://champion.gg/champion/{}/{}?league=", "https://champion.gg/champion/{}/{}?league=",
id, position id, position
)) ))
.send() .call()
.unwrap(); .unwrap();
if req.status().is_success() { let response_status = req.status();
if 300 > response_status && response_status >= 200 {
lazy_static! { lazy_static! {
static ref RE: Regex = static ref RE: Regex =
Regex::new(r"(?m)^\s+matchupData\.championData = (.*)$").unwrap(); Regex::new(r"(?m)^\s+matchupData\.championData = (.*)$").unwrap();
} }
let data: Value = serde_json::from_str(&RE.captures(&req.text().unwrap())?[1]).unwrap(); let data: Value =
serde_json::from_str(&RE.captures(&req.into_string().unwrap())?[1]).unwrap();
let mut blocks = vec![]; let mut blocks = vec![];
for (label, path) in ITEM_TYPES.iter() { for (label, path) in ITEM_TYPES.iter() {
if !data[&path[0]].get(&path[1]).is_none() { if !data[&path[0]].get(&path[1]).is_none() {

View file

@ -35,14 +35,14 @@ pub trait DataSource {
fn get_champs_with_positions_and_patch( fn get_champs_with_positions_and_patch(
&self, &self,
client: &reqwest::blocking::Client, client: &ureq::Agent,
) -> (IndexMap<String, Vec<String>>, String); ) -> (IndexMap<String, Vec<String>>, String);
fn get_champ_data_with_win_pourcentage( fn get_champ_data_with_win_pourcentage(
&self, &self,
id: &str, id: &str,
position: &str, position: &str,
client: &reqwest::blocking::Client, client: &ureq::Agent,
) -> Option<(Vec<Value>, f64)>; ) -> Option<(Vec<Value>, f64)>;
fn write_item_set( fn write_item_set(
@ -52,7 +52,7 @@ pub trait DataSource {
pos: &str, pos: &str,
ver: &str, ver: &str,
path: &PathBuf, path: &PathBuf,
client: &reqwest::blocking::Client, client: &ureq::Agent,
) { ) {
info!("Retrieving data for {} at {}", name, pos); info!("Retrieving data for {} at {}", name, pos);
let data = self.get_champ_data_with_win_pourcentage(id, pos, client); let data = self.get_champ_data_with_win_pourcentage(id, pos, client);

View file

@ -1,4 +1,6 @@
use crate::data_source::{Build, DataSource, Item}; use crate::data_source::{Build, DataSource, Item};
#[cfg(test)]
use crate::time::Duration;
use indexmap::IndexMap; use indexmap::IndexMap;
use regex::Regex; use regex::Regex;
use serde_derive::Deserialize; use serde_derive::Deserialize;
@ -88,7 +90,7 @@ struct Summoner {
} }
impl KBDataSource { impl KBDataSource {
pub fn new(client: &reqwest::blocking::Client) -> KBDataSource { pub fn new(client: &ureq::Agent) -> KBDataSource {
let mut datasource = KBDataSource { let mut datasource = KBDataSource {
token: None, token: None,
internal_classname_mapping: IndexMap::new(), internal_classname_mapping: IndexMap::new(),
@ -99,9 +101,9 @@ impl KBDataSource {
} }
// It will be better to use Result... // It will be better to use Result...
fn get_auth_token(&self, client: &reqwest::blocking::Client) -> Option<String> { fn get_auth_token(&self, client: &ureq::Agent) -> Option<String> {
let bundle = match client.get("https://koreanbuilds.net/bundle.js").send() { let bundle = match client.get("https://koreanbuilds.net/bundle.js").call() {
Ok(resp) => match resp.text() { Ok(resp) => match resp.into_string() {
Ok(val) => val, Ok(val) => val,
Err(_) => return None, Err(_) => return None,
}, },
@ -121,7 +123,7 @@ impl KBDataSource {
}; };
} }
fn get_classname_mapping(&self, client: &reqwest::blocking::Client) -> IndexMap<String, String> { fn get_classname_mapping(&self, client: &ureq::Agent) -> IndexMap<String, String> {
let mut mapping = IndexMap::new(); let mut mapping = IndexMap::new();
match self.get_champion_response(client) { match self.get_champion_response(client) {
Some(data) => { Some(data) => {
@ -134,18 +136,18 @@ impl KBDataSource {
mapping mapping
} }
fn get_champion_response(&self, client: &reqwest::blocking::Client) -> Option<ChampionResponse> { fn get_champion_response(&self, client: &ureq::Agent) -> Option<ChampionResponse> {
let token = match self.token.clone() { let token = match self.token.clone() {
Some(t) => t, Some(t) => t,
None => return None, None => return None,
}; };
match client match client
.get("https://api.koreanbuilds.net/champions?patchid=-1") .get("https://api.koreanbuilds.net/champions?patchid=-1")
.header("Accept", "application/json") .set("Accept", "application/json")
.header("Authorization", token) .set("Authorization", token.as_str())
.send() .call()
{ {
Ok(resp) => match resp.json() { Ok(resp) => match resp.into_json() {
Ok(val) => return val, Ok(val) => return val,
Err(_) => return None, Err(_) => return None,
}, },
@ -186,7 +188,7 @@ impl DataSource for KBDataSource {
fn get_champs_with_positions_and_patch( fn get_champs_with_positions_and_patch(
&self, &self,
client: &reqwest::blocking::Client, client: &ureq::Agent,
) -> (IndexMap<String, Vec<String>>, String) { ) -> (IndexMap<String, Vec<String>>, String) {
let mut champions = IndexMap::new(); let mut champions = IndexMap::new();
let data: ChampionResponse = match self.get_champion_response(client) { let data: ChampionResponse = match self.get_champion_response(client) {
@ -209,7 +211,7 @@ impl DataSource for KBDataSource {
&self, &self,
id: &str, id: &str,
position: &str, position: &str,
client: &reqwest::blocking::Client, client: &ureq::Agent,
) -> Option<(Vec<Value>, f64)> { ) -> Option<(Vec<Value>, f64)> {
let token = match self.token.clone() { let token = match self.token.clone() {
Some(t) => t, Some(t) => t,
@ -224,11 +226,11 @@ impl DataSource for KBDataSource {
"https://api.koreanbuilds.net/builds?chmpname={}&patchid=-2&position={}", "https://api.koreanbuilds.net/builds?chmpname={}&patchid=-2&position={}",
map_id, position map_id, position
)) ))
.header("Accept", "application/json") .set("Accept", "application/json")
.header("Authorization", token) .set("Authorization", token.as_str())
.send() .call()
{ {
Ok(resp) => match resp.json() { Ok(resp) => match resp.into_json() {
Ok(val) => val, Ok(val) => val,
Err(_) => { Err(_) => {
return None; return None;
@ -349,10 +351,9 @@ mod tests {
token: None, token: None,
internal_classname_mapping: IndexMap::new(), internal_classname_mapping: IndexMap::new(),
}; };
let client = reqwest::blocking::Client::builder() let client = ureq::AgentBuilder::new()
.timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(10))
.build() .build();
.unwrap();
match datasource.get_auth_token(&client) { match datasource.get_auth_token(&client) {
Some(token) => assert!(token.len() > 0), Some(token) => assert!(token.len() > 0),
None => assert!(false), None => assert!(false),
@ -361,10 +362,9 @@ mod tests {
#[test] #[test]
fn test_get_champs_with_positions_and_patch() { fn test_get_champs_with_positions_and_patch() {
let client = reqwest::blocking::Client::builder() let client = ureq::AgentBuilder::new()
.timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(10))
.build() .build();
.unwrap();
let datasource = KBDataSource::new(&client); let datasource = KBDataSource::new(&client);
let champs_with_positions_and_patch = let champs_with_positions_and_patch =
datasource.get_champs_with_positions_and_patch(&client); datasource.get_champs_with_positions_and_patch(&client);
@ -373,10 +373,9 @@ mod tests {
#[test] #[test]
fn test_get_champ_data_with_win_pourcentage() { fn test_get_champ_data_with_win_pourcentage() {
let client = reqwest::blocking::Client::builder() let client = ureq::AgentBuilder::new()
.timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(10))
.build() .build();
.unwrap();
let datasource = KBDataSource::new(&client); let datasource = KBDataSource::new(&client);
let result = datasource.get_champ_data_with_win_pourcentage("Aatrox", "TOP", &client); let result = datasource.get_champ_data_with_win_pourcentage("Aatrox", "TOP", &client);
assert!(result.is_some()); assert!(result.is_some());

View file

@ -1,9 +1,10 @@
use indexmap::IndexMap; use indexmap::IndexMap;
use log::{error, info, LevelFilter}; use log::{error, info, LevelFilter};
use reqwest::header;
use serde_derive::Deserialize; use serde_derive::Deserialize;
use simple_logger::SimpleLogger; use simple_logger::SimpleLogger;
use std::io::{Error, ErrorKind}; use std::io::Error;
#[cfg(target_os = "windows")]
use std::io::ErrorKind;
use std::path::PathBuf; use std::path::PathBuf;
use std::{fs, thread, time}; use std::{fs, thread, time};
use time::Duration; use time::Duration;
@ -35,13 +36,15 @@ struct ChampInfo {
name: String, name: String,
} }
const USER_AGENT: &str = const USER_AGENT_KEY: &str = "User-Agent";
const USER_AGENT_VALUE: &str =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0"; "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:60.0) Gecko/20100101 Firefox/60.0";
const LOL_CHAMPS_DIR: &str = ".\\champs"; const LOL_CHAMPS_DIR: &str = ".\\champs";
fn main() { fn main() {
SimpleLogger::new() SimpleLogger::new()
.with_level(LevelFilter::Info) .with_level(LevelFilter::Info)
.with_module_level("ureq", LevelFilter::Error)
.init() .init()
.unwrap(); .unwrap();
@ -53,23 +56,16 @@ fn main() {
}; };
info!("LoL Champs Folder: {}", lol_champs_dir.to_str().unwrap()); info!("LoL Champs Folder: {}", lol_champs_dir.to_str().unwrap());
let mut headers = header::HeaderMap::new(); let client = ureq::AgentBuilder::new()
headers.insert(
header::USER_AGENT,
header::HeaderValue::from_static(USER_AGENT),
);
let client = reqwest::blocking::Client::builder()
.default_headers(headers)
.timeout(Duration::from_secs(10)) .timeout(Duration::from_secs(10))
.build() .build();
.unwrap();
let realm: Realm = client let realm: Realm = client
.get("https://ddragon.leagueoflegends.com/realms/euw.json") .get("https://ddragon.leagueoflegends.com/realms/euw.json")
.send() .set(USER_AGENT_KEY, USER_AGENT_VALUE)
.call()
.unwrap() .unwrap()
.json() .into_json()
.unwrap(); .unwrap();
info!("LoL version: {}", realm.v); info!("LoL version: {}", realm.v);
@ -78,9 +74,10 @@ fn main() {
"https://ddragon.leagueoflegends.com/cdn/{}/data/en_US/champion.json", "https://ddragon.leagueoflegends.com/cdn/{}/data/en_US/champion.json",
realm.v realm.v
)) ))
.send() .set(USER_AGENT_KEY, USER_AGENT_VALUE)
.call()
.unwrap() .unwrap()
.json() .into_json()
.unwrap(); .unwrap();
info!("LoL numbers of champs: {}", champion.data.len()); info!("LoL numbers of champs: {}", champion.data.len());

View file

@ -11,7 +11,7 @@ impl DataSource for PBDataSource {
fn get_champs_with_positions_and_patch( fn get_champs_with_positions_and_patch(
&self, &self,
_client: &reqwest::blocking::Client, _client: &ureq::Agent,
) -> (IndexMap<String, Vec<String>>, String) { ) -> (IndexMap<String, Vec<String>>, String) {
(IndexMap::new(), String::new()) (IndexMap::new(), String::new())
} }
@ -20,7 +20,7 @@ impl DataSource for PBDataSource {
&self, &self,
_id: &str, _id: &str,
_position: &str, _position: &str,
_client: &reqwest::blocking::Client, _client: &ureq::Agent,
) -> Option<(Vec<Value>, f64)> { ) -> Option<(Vec<Value>, f64)> {
None None
} }