alvaro/lol-analytics
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

move storage and api to apps, create base analyzer

Browse Source
This commit is contained in:
Álvaro
2024-05-10 06:08:25 +02:00
parent 987339f517
commit 2e56c7105b
42 changed files with 403 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/scrapper/lib/scrapper/api/account_api.ex
-24
View File
@@ -1,24 +0,0 @@
defmodule Scrapper.Api.AccountApi do
@get_puuid_endpoint "https://europe.api.riotgames.com/riot/account/v1/accounts/by-riot-id/%{gameName}/%{tagLine}"
@spec get_puuid(String.t(), String.t()) :: {:ok, String.t()} | {:error, String.t()}
def get_puuid(name, tag) do
url =
@get_puuid_endpoint
|> String.replace("%{gameName}", name)
|> String.replace("%{tagLine}", tag)
api_key = System.get_env("RIOT_API_KEY")
headers = [{"X-Riot-Token", api_key}]
response = HTTPoison.get!(url, headers, timeout: 5000)
case response.status_code do
200 ->
{:ok, Poison.decode(response.body)}
code ->
Logger.error("Error getting puuid from player #{name} \##{tag}")
{:err, response.status_code}
end
end
end
apps/scrapper/lib/scrapper/api/match_api.ex
-51
View File
@@ -1,51 +0,0 @@
defmodule Scrapper.Data.Api.MatchApi do
import Logger
@match_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/%{matchid}"
@puuid_matches_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/by-puuid/%{puuid}/ids"
@doc """
Get match by id
iex> Scrapper.Data.MatchApi.get_match_by_id("EUW1_6921743825")
"""
@spec get_match_by_id(String.t()) :: %Scrapper.Api.Model.MatchResponse{}
def get_match_by_id(match_id) do
url = String.replace(@match_base_endpoint, "%{matchid}", match_id)
Logger.info("Making request to #{url}")
api_key = System.get_env("RIOT_API_KEY")
headers = [{"X-Riot-Token", api_key}]
response = HTTPoison.get!(url, headers, timeout: 5000)
case response.status_code do
200 ->
{:ok, response.body}
_ ->
Logger.error("Error getting match by id: #{match_id} #{inspect(response)}")
{:err, response.status_code}
end
end
@doc """
Get matches from player
iex> Scrapper.Data.Api.MatchApi.get_matches_from_player "JB6TdEWlKjZwnbgdSzOogYepNfjLPdUh68S8b4kUu4EEZy4R4MMAgv92QMj1XgVjtzHmZVLaOW7mzg"
"""
@spec get_matches_from_player(String.t()) :: list(String.t()) | integer()
def get_matches_from_player(puuid) do
url = String.replace(@puuid_matches_base_endpoint, "%{puuid}", URI.encode(puuid))
Logger.info("Making request to #{url}")
api_key = System.get_env("RIOT_API_KEY")
headers = [{"X-Riot-Token", api_key}]
response = HTTPoison.get!(url, headers, timeout: 5000)
case response.status_code do
200 ->
{:ok, Poison.decode!(response.body)}
code ->
Logger.error("Error getting matches from player #{puuid} #{code}")
{:err, response.status_code}
end
end
end
apps/scrapper/lib/scrapper/api/model/info.ex
-20
View File
@@ -1,20 +0,0 @@
defmodule Scrapper.Api.Model.Info do
alias Scrapper.Api.Model.Participant
defstruct endOfGameResult: "",
gameCreation: "",
gameDuration: "",
gameEndTimestamp: "",
gameId: "",
gameMode: "",
gameName: "",
gameStartTimestamp: "",
gameType: "",
gameVersion: "",
mapId: "",
participants: [%Participant{}],
platformId: "",
queueId: "",
teams: "",
tournamentCode: ""
end
apps/scrapper/lib/scrapper/api/model/match_response.ex
-6
View File
@@ -1,6 +0,0 @@
defmodule Scrapper.Api.Model.MatchResponse do
alias Scrapper.Api.Model.{Info, Metadata}
defstruct metadata: %Metadata{},
info: %Info{}
end
apps/scrapper/lib/scrapper/api/model/metadata.ex
-3
View File
@@ -1,3 +0,0 @@
defmodule Scrapper.Api.Model.Metadata do
defstruct [:dataVersion, :matchId, :participants]
end
apps/scrapper/lib/scrapper/api/model/participant.ex
-135
View File
@@ -1,135 +0,0 @@
defmodule Scrapper.Api.Model.Participant do
# Enum.map(participant, fn {k,_v} -> ":#{k}" end) |> Enum.join(", ")
defstruct [
:onMyWayPings,
:totalDamageDealt,
:summoner1Casts,
:totalEnemyJungleMinionsKilled,
:summoner2Casts,
:totalTimeCCDealt,
:eligibleForProgression,
:enemyVisionPings,
:assists,
:teamPosition,
:objectivesStolenAssists,
:perks,
:spell3Casts,
:totalHeal,
:doubleKills,
:missions,
:physicalDamageDealt,
:summonerName,
:champExperience,
:quadraKills,
:neutralMinionsKilled,
:basicPings,
:pushPings,
:playerAugment2,
:wardsPlaced,
:individualPosition,
:damageSelfMitigated,
:dangerPings,
:largestMultiKill,
:puuid,
:subteamPlacement,
:turretsLost,
:role,
:visionClearedPings,
:goldSpent,
:inhibitorTakedowns,
:summoner2Id,
:trueDamageDealtToChampions,
:needVisionPings,
:champLevel,
:championTransform,
:bountyLevel,
:teamEarlySurrendered,
:championName,
:largestKillingSpree,
:gameEndedInSurrender,
:summoner1Id,
:getBackPings,
:nexusKills,
:baronKills,
:item6,
:firstTowerKill,
:summonerLevel,
:damageDealtToTurrets,
:commandPings,
:totalHealsOnTeammates,
:turretTakedowns,
:playerSubteamId,
:longestTimeSpentLiving,
:item0,
:summonerId,
:assistMePings,
:wardsKilled,
:physicalDamageTaken,
:magicDamageDealt,
:timePlayed,
:item2,
:firstBloodKill,
:goldEarned,
:magicDamageDealtToChampions,
:item1,
:nexusLost,
:itemsPurchased,
:tripleKills,
:sightWardsBoughtInGame,
:placement,
:consumablesPurchased,
:item5,
:totalDamageTaken,
:item4,
:playerAugment4,
:physicalDamageDealtToChampions,
:spell1Casts,
:totalTimeSpentDead,
:nexusTakedowns,
:gameEndedInEarlySurrender,
:dragonKills,
:totalAllyJungleMinionsKilled,
:killingSprees,
:detectorWardsPlaced,
:trueDamageDealt,
:damageDealtToObjectives,
:damageDealtToBuildings,
:totalDamageDealtToChampions,
:lane,
:totalMinionsKilled,
:playerAugment3,
:spell2Casts,
:pentaKills,
:firstTowerAssist,
:enemyMissingPings,
:turretKills,
:championId,
:trueDamageTaken,
:deaths,
:win,
:magicDamageTaken,
:item3,
:riotIdGameName,
:firstBloodAssist,
:profileIcon,
:inhibitorsLost,
:visionScore,
:playerAugment1,
:allInPings,
:largestCriticalStrike,
:inhibitorKills,
:riotIdTagline,
:unrealKills,
:totalDamageShieldedOnTeammates,
:visionWardsBoughtInGame,
:holdPings,
:participantId,
:kills,
:challenges,
:objectivesStolen,
:spell4Casts,
:totalUnitsHealed,
:teamId,
:timeCCingOthers
]
end
apps/scrapper/lib/scrapper/match_classifier.ex
+31
View File
@@ -0,0 +1,31 @@
defmodule Scrapper.MatchClassifier do
require Logger
@spec classify_match(%LoLAPI.Model.MatchResponse{}) :: nil
def classify_match(match = %LoLAPI.Model.MatchResponse{}) do
classify_match_by_queue(match.info.queueId)
end
@spec classify_match_by_queue(String.t()) :: nil
def classify_match_by_queue("420") do
matches = Storage.MatchStorage.S3MatchStorage.list_matches()
total_matches = Enum.count(matches)
matches
|> Enum.with_index(fn match, index -> {match, index} end)
|> Scrapper.Parallel.peach(fn {match, index} ->
%{key: json_file} = match
[key | _] = String.split(json_file, ".")
Logger.info("Match at #{index} of #{total_matches} is classified")
response = HTTPoison.get!("http://localhost:9000/matches/#{key}.json", [], timeout: 5000)
%{"info" => %{"gameVersion" => gameVersion}} = Poison.decode!(response.body)
Storage.MatchStorage.S3MatchStorage.store_match(key, response.body, "ranked", gameVersion)
match
end)
end
# pass functions, not data
def classify_match_by_queue(_) do
end
end
apps/scrapper/lib/scrapper/parallel/Parallel.ex
+7
View File
@@ -0,0 +1,7 @@
defmodule Scrapper.Parallel do
def peach(enum, fun, concurrency \\ 10, timeout \\ :infinity) do
Task.async_stream(enum, &fun.(&1), max_concurrency: concurrency, timeout: timeout)
|> Stream.each(fn {:ok, val} -> val end)
|> Enum.to_list()
end
end
apps/scrapper/lib/scrapper/processor/match_processor.ex
+6 -4
View File
@@ -20,7 +20,7 @@ defmodule Scrapper.Processor.MatchProcessor do
]},
concurrency: 1,
rate_limiting: [
interval: 1000 * 1,
interval: 333 * 1,
allowed_messages: 1
]
],
@@ -36,15 +36,15 @@ defmodule Scrapper.Processor.MatchProcessor do
def handle_message(_, message = %Broadway.Message{}, _) do
match_id = message.data
resp = Scrapper.Data.Api.MatchApi.get_match_by_id(match_id)
resp = LoLAPI.MatchApi.get_match_by_id(match_id)
process_resp(resp, match_id)
message
end
def process_resp({:ok, raw_match}, match_id) do
decoded_match = Poison.decode!(raw_match, as: %Scrapper.Api.Model.MatchResponse{})
match_url = Scrapper.Storage.S3MatchStorage.store_match(match_id, raw_match)
decoded_match = Poison.decode!(raw_match, as: %LoLAPI.Model.MatchResponse{})
match_url = Storage.MatchStorage.S3MatchStorage.store_match(match_id, raw_match)
match = LolAnalytics.Match.MatchRepo.get_match(match_id)
case match do
@@ -59,6 +59,8 @@ defmodule Scrapper.Processor.MatchProcessor do
end
decoded_match.metadata.participants
|> Enum.shuffle()
|> Enum.take(2)
|> Enum.each(fn participant_puuid ->
Scrapper.Queue.PlayerQueue.queue_puuid(participant_puuid)
end)
apps/scrapper/lib/scrapper/processor/player_processor.ex
+1 -1
View File
@@ -46,7 +46,7 @@ defmodule Scrapper.Processor.PlayerProcessor do
update_player_processed(player)
end
match_history = Scrapper.Data.Api.MatchApi.get_matches_from_player(puuid)
match_history = LoLAPI.MatchApi.get_matches_from_player(puuid)
case match_history do
{:ok, matches} ->
apps/scrapper/lib/scrapper/storage/match_storage.ex
-4
View File
@@ -1,4 +0,0 @@
defmodule Scrapper.Data.Storage.MatchStorage do
@callback get_match(String.t()) :: {:ok, Scrapper.Data.Match.t()} | {:error, :not_found}
@callback save_match(String.t(), Scrapper.Data.Match.t()) :: :ok
end
apps/scrapper/lib/scrapper/storage/s3_match_storage.ex
-67
View File
@@ -1,67 +0,0 @@
defmodule Scrapper.Storage.S3MatchStorage do
require Logger
import SweetXml
@behaviour Scrapper.Data.Storage.MatchStorage
def get_match(match_id) do
""
end
# check for to get all pages next_continuation_token
def list_matches() do
{:ok, %{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} =
ExAws.S3.list_objects_v2("matches")
|> ExAws.request()
if next_continuation_token do
list_matches(contents, next_continuation_token)
# |> Enum.map(fn %{key: key} -> key end)
else
contents
# |> Enum.map(fn %{key: key} -> key end)
end
end
@spec list_matches(list(String.t()), String.t()) :: list(String.t())
def list_matches(acc, continuation_token) do
resp =
{:ok,
%{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} =
ExAws.S3.list_objects_v2("matches", continuation_token: continuation_token)
|> ExAws.request()
if next_continuation_token == "" do
acc ++ contents
else
list_matches(acc ++ contents, next_continuation_token)
end
end
def download_match(destination_path, url) do
ExAws.S3.download_file(url, destination_path)
|> ExAws.request()
end
@doc """
iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content"
"""
@spec store_match(String.t(), String.t()) :: none()
def store_match(match_id, match_data) do
File.write("/tmp/#{match_id}.json", match_data)
url =
"/tmp/#{match_id}.json"
|> ExAws.S3.Upload.stream_file()
|> ExAws.S3.upload("matches", "#{match_id}.json")
|> ExAws.request!()
|> extract_s3_url_from_upload
Logger.info("Stored match at #{url}")
url
end
defp extract_s3_url_from_upload(%{:body => %{:location => location}} = _s3_response),
do: location
end
apps/scrapper/mix.exs
+3 -6
View File
@@ -26,15 +26,12 @@ defmodule Scrapper.MixProject do
# Run "mix help deps" to learn about dependencies.
defp deps do
[
{:httpoison, "~> 2.2"},
{:poison, "~> 5.0"},
{:ex_aws, "~> 2.1"},
{:ex_aws_s3, "~> 2.5.3"},
{:hackney, "~> 1.9"},
{:sweet_xml, "~> 0.6"},
{:broadway_rabbitmq, "~> 0.7"},
{:amqp, "~> 3.3"},
{:lol_analytics, in_umbrella: true}
{:lol_analytics, in_umbrella: true},
{:lol_api, in_umbrella: true},
{:storage, in_umbrella: true}
# {:dep_from_hexpm, "~> 0.3.0"},
# {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"},
# {:sibling_app_in_umbrella, in_umbrella: true}