From 2e56c7105bf495a3e5099c4bee84e459c4824cdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro?= Date: Fri, 10 May 2024 06:08:25 +0200 Subject: [PATCH] move storage and api to apps, create base analyzer --- .gitignore | 2 +- .../lol_analytics/analyzer/base_analyzer.ex | 3 ++ .../analyzer/champion_analyzer.ex | 32 ++++++++++++++ .../lib/lol_analytics/player/player_repo.ex | 4 +- apps/lol_analytics/mix.exs | 5 ++- .../repo/migrations/20240502203545_player.exs | 1 + apps/lol_api/.formatter.exs | 4 ++ apps/lol_api/.gitignore | 26 +++++++++++ apps/lol_api/README.md | 21 +++++++++ apps/lol_api/config/config.exs | 4 ++ .../lib}/api/account_api.ex | 4 +- .../scrapper => lol_api/lib}/api/match_api.ex | 10 ++--- .../lib}/api/model/info.ex | 4 +- apps/lol_api/lib/api/model/match_response.ex | 6 +++ .../lib}/api/model/metadata.ex | 2 +- .../lib}/api/model/participant.ex | 2 +- apps/lol_api/lib/lol_api.ex | 18 ++++++++ apps/lol_api/mix.exs | 35 +++++++++++++++ apps/lol_api/test/lo_lapi_test.exs | 8 ++++ apps/lol_api/test/test_helper.exs | 1 + .../lib/scrapper/api/model/match_response.ex | 6 --- .../scrapper/lib/scrapper/match_classifier.ex | 31 +++++++++++++ .../lib/scrapper/parallel/Parallel.ex | 7 +++ .../lib/scrapper/processor/match_processor.ex | 10 +++-- .../scrapper/processor/player_processor.ex | 2 +- .../lib/scrapper/storage/match_storage.ex | 4 -- apps/scrapper/mix.exs | 9 ++-- apps/storage/.formatter.exs | 4 ++ apps/storage/.gitignore | 26 +++++++++++ apps/storage/README.md | 21 +++++++++ apps/storage/config/config.exs | 3 ++ apps/storage/config/dev.exs | 3 ++ apps/storage/config/libs/ex_aws_dev.exs | 17 ++++++++ apps/storage/config/prod.exs | 1 + apps/storage/config/test.exs | 0 .../lib/match_storage/match_storage.ex | 6 +++ .../lib/match_storage}/s3_match_storage.ex | 43 +++++++++++++------ apps/storage/lib/storage.ex | 18 ++++++++ apps/storage/mix.exs | 37 ++++++++++++++++ apps/storage/test/storage_test.exs | 8 ++++ apps/storage/test/test_helper.exs | 1 + config/config.exs | 2 + 42 files changed, 403 insertions(+), 48 deletions(-) create mode 100644 apps/lol_analytics/lib/lol_analytics/analyzer/base_analyzer.ex create mode 100644 apps/lol_analytics/lib/lol_analytics/analyzer/champion_analyzer.ex create mode 100644 apps/lol_api/.formatter.exs create mode 100644 apps/lol_api/.gitignore create mode 100644 apps/lol_api/README.md create mode 100644 apps/lol_api/config/config.exs rename apps/{scrapper/lib/scrapper => lol_api/lib}/api/account_api.ex (93%) rename apps/{scrapper/lib/scrapper => lol_api/lib}/api/match_api.ex (81%) rename apps/{scrapper/lib/scrapper => lol_api/lib}/api/model/info.ex (86%) create mode 100644 apps/lol_api/lib/api/model/match_response.ex rename apps/{scrapper/lib/scrapper => lol_api/lib}/api/model/metadata.ex (57%) rename apps/{scrapper/lib/scrapper => lol_api/lib}/api/model/participant.ex (98%) create mode 100644 apps/lol_api/lib/lol_api.ex create mode 100644 apps/lol_api/mix.exs create mode 100644 apps/lol_api/test/lo_lapi_test.exs create mode 100644 apps/lol_api/test/test_helper.exs delete mode 100644 apps/scrapper/lib/scrapper/api/model/match_response.ex create mode 100644 apps/scrapper/lib/scrapper/match_classifier.ex create mode 100644 apps/scrapper/lib/scrapper/parallel/Parallel.ex delete mode 100644 apps/scrapper/lib/scrapper/storage/match_storage.ex create mode 100644 apps/storage/.formatter.exs create mode 100644 apps/storage/.gitignore create mode 100644 apps/storage/README.md create mode 100644 apps/storage/config/config.exs create mode 100644 apps/storage/config/dev.exs create mode 100644 apps/storage/config/libs/ex_aws_dev.exs create mode 100644 apps/storage/config/prod.exs create mode 100644 apps/storage/config/test.exs create mode 100644 apps/storage/lib/match_storage/match_storage.ex rename apps/{scrapper/lib/scrapper/storage => storage/lib/match_storage}/s3_match_storage.ex (61%) create mode 100644 apps/storage/lib/storage.ex create mode 100644 apps/storage/mix.exs create mode 100644 apps/storage/test/storage_test.exs create mode 100644 apps/storage/test/test_helper.exs diff --git a/.gitignore b/.gitignore index 6c10cc9..e5334ca 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,4 @@ erl_crash.dump .env - +.DS_store diff --git a/apps/lol_analytics/lib/lol_analytics/analyzer/base_analyzer.ex b/apps/lol_analytics/lib/lol_analytics/analyzer/base_analyzer.ex new file mode 100644 index 0000000..1c62152 --- /dev/null +++ b/apps/lol_analytics/lib/lol_analytics/analyzer/base_analyzer.ex @@ -0,0 +1,3 @@ +defmodule LolAnalytics.Analyzer.BaseAnalyzer do + @callback analyze(:url, path :: String.t()) :: :ok +end diff --git a/apps/lol_analytics/lib/lol_analytics/analyzer/champion_analyzer.ex b/apps/lol_analytics/lib/lol_analytics/analyzer/champion_analyzer.ex new file mode 100644 index 0000000..4be0b3f --- /dev/null +++ b/apps/lol_analytics/lib/lol_analytics/analyzer/champion_analyzer.ex @@ -0,0 +1,32 @@ +defmodule LolAnalytics.Analyzer.ChampionAnalyzer do + alias Hex.HTTP + @behaviour LolAnalytics.Analyzer.BaseAnalyzer + + @doc """ + iex> LolAnalytics.Analyzer.ChampionAnalyzer.analyze(:url, "https://na1.api.riotgames.com/lol/match/v4/match/234567890123456789") + :ok + """ + @impl true + @spec analyze(atom(), String.t()) :: :ok + def analyze(:url, path) do + data = HTTPoison.get!(path) + analyze(:data, data.body) + :ok + end + + @doc """ + iex> LolAnalytics.Analyzer.ChampionAnalyzer.analyze(:url, "http://localhost:9000/ranked/14.9.580.2108/EUW1_6923309745.json") + """ + @impl true + @spec analyze(atom(), any()) :: list(LoLAPI.Model.Participant.t()) + def analyze(:data, data) do + decoded = Poison.decode!(data) + + %{"info" => %{"participants" => participants}} = decoded + + participants + |> Enum.each(fn %{"win" => win, "championId" => champion_id} -> + IO.inspect(%{win: win, champion_id: champion_id}) + end) + end +end diff --git a/apps/lol_analytics/lib/lol_analytics/player/player_repo.ex b/apps/lol_analytics/lib/lol_analytics/player/player_repo.ex index b2f24a8..f4f5282 100644 --- a/apps/lol_analytics/lib/lol_analytics/player/player_repo.ex +++ b/apps/lol_analytics/lib/lol_analytics/player/player_repo.ex @@ -15,7 +15,9 @@ defmodule LolAnalytics.Player.PlayerRepo do def get_player(puuid) do query = from p in PlayerSchema, where: p.puuid == ^puuid - LoLAnalytics.Repo.one(query) + + LoLAnalytics.Repo.all(query) + |> List.first() end @spec insert_player(String.t(), keyword()) :: %PlayerSchema{} diff --git a/apps/lol_analytics/mix.exs b/apps/lol_analytics/mix.exs index cb0915f..8d4a25d 100644 --- a/apps/lol_analytics/mix.exs +++ b/apps/lol_analytics/mix.exs @@ -40,7 +40,10 @@ defmodule LoLAnalytics.MixProject do {:phoenix_pubsub, "~> 2.1"}, {:ecto_sql, "~> 3.10"}, {:postgrex, ">= 0.0.0"}, - {:jason, "~> 1.2"} + {:jason, "~> 1.2"}, + {:lol_api, in_umbrella: true}, + {:httpoison, "~> 2.2"}, + {:poison, "~> 5.0"} ] end diff --git a/apps/lol_analytics/priv/repo/migrations/20240502203545_player.exs b/apps/lol_analytics/priv/repo/migrations/20240502203545_player.exs index 1445f1a..ecef8fa 100644 --- a/apps/lol_analytics/priv/repo/migrations/20240502203545_player.exs +++ b/apps/lol_analytics/priv/repo/migrations/20240502203545_player.exs @@ -10,6 +10,7 @@ defmodule LoLAnalytics.Repo.Migrations.Player do timestamps() end + unique_index("player", :puuid) create index(:player, [:puuid]) end end diff --git a/apps/lol_api/.formatter.exs b/apps/lol_api/.formatter.exs new file mode 100644 index 0000000..d2cda26 --- /dev/null +++ b/apps/lol_api/.formatter.exs @@ -0,0 +1,4 @@ +# Used by "mix format" +[ + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/apps/lol_api/.gitignore b/apps/lol_api/.gitignore new file mode 100644 index 0000000..7229561 --- /dev/null +++ b/apps/lol_api/.gitignore @@ -0,0 +1,26 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +lol_api-*.tar + +# Temporary files, for example, from tests. +/tmp/ diff --git a/apps/lol_api/README.md b/apps/lol_api/README.md new file mode 100644 index 0000000..d01c95a --- /dev/null +++ b/apps/lol_api/README.md @@ -0,0 +1,21 @@ +# LoLAPI + +**TODO: Add description** + +## Installation + +If [available in Hex](https://hex.pm/docs/publish), the package can be installed +by adding `lol_api` to your list of dependencies in `mix.exs`: + +```elixir +def deps do + [ + {:lol_api, "~> 0.1.0"} + ] +end +``` + +Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) +and published on [HexDocs](https://hexdocs.pm). Once published, the docs can +be found at . + diff --git a/apps/lol_api/config/config.exs b/apps/lol_api/config/config.exs new file mode 100644 index 0000000..bd778c3 --- /dev/null +++ b/apps/lol_api/config/config.exs @@ -0,0 +1,4 @@ +import Config + +config :lol_api, + riot_api_key: System.get_env("RIOT_API_KEY") diff --git a/apps/scrapper/lib/scrapper/api/account_api.ex b/apps/lol_api/lib/api/account_api.ex similarity index 93% rename from apps/scrapper/lib/scrapper/api/account_api.ex rename to apps/lol_api/lib/api/account_api.ex index 8ee0f70..59e203e 100644 --- a/apps/scrapper/lib/scrapper/api/account_api.ex +++ b/apps/lol_api/lib/api/account_api.ex @@ -1,4 +1,6 @@ -defmodule Scrapper.Api.AccountApi do +defmodule LoLAPI.AccountApi do + require Logger + @get_puuid_endpoint "https://europe.api.riotgames.com/riot/account/v1/accounts/by-riot-id/%{gameName}/%{tagLine}" @spec get_puuid(String.t(), String.t()) :: {:ok, String.t()} | {:error, String.t()} diff --git a/apps/scrapper/lib/scrapper/api/match_api.ex b/apps/lol_api/lib/api/match_api.ex similarity index 81% rename from apps/scrapper/lib/scrapper/api/match_api.ex rename to apps/lol_api/lib/api/match_api.ex index 04232df..25500bd 100644 --- a/apps/scrapper/lib/scrapper/api/match_api.ex +++ b/apps/lol_api/lib/api/match_api.ex @@ -1,14 +1,14 @@ -defmodule Scrapper.Data.Api.MatchApi do - import Logger +defmodule LoLAPI.MatchApi do + require Logger @match_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/%{matchid}" @puuid_matches_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/by-puuid/%{puuid}/ids" @doc """ Get match by id - iex> Scrapper.Data.MatchApi.get_match_by_id("EUW1_6921743825") + iex> LoLAPI.MatchApi.get_match_by_id("EUW1_6921743825") """ - @spec get_match_by_id(String.t()) :: %Scrapper.Api.Model.MatchResponse{} + @spec get_match_by_id(String.t()) :: %LoLAPI.Model.MatchResponse{} def get_match_by_id(match_id) do url = String.replace(@match_base_endpoint, "%{matchid}", match_id) Logger.info("Making request to #{url}") @@ -29,7 +29,7 @@ defmodule Scrapper.Data.Api.MatchApi do @doc """ Get matches from player - iex> Scrapper.Data.Api.MatchApi.get_matches_from_player "JB6TdEWlKjZwnbgdSzOogYepNfjLPdUh68S8b4kUu4EEZy4R4MMAgv92QMj1XgVjtzHmZVLaOW7mzg" + iex> LoLAPI.MatchApi.get_matches_from_player "JB6TdEWlKjZwnbgdSzOogYepNfjLPdUh68S8b4kUu4EEZy4R4MMAgv92QMj1XgVjtzHmZVLaOW7mzg" """ @spec get_matches_from_player(String.t()) :: list(String.t()) | integer() def get_matches_from_player(puuid) do diff --git a/apps/scrapper/lib/scrapper/api/model/info.ex b/apps/lol_api/lib/api/model/info.ex similarity index 86% rename from apps/scrapper/lib/scrapper/api/model/info.ex rename to apps/lol_api/lib/api/model/info.ex index 6115fe7..1e8b2bc 100644 --- a/apps/scrapper/lib/scrapper/api/model/info.ex +++ b/apps/lol_api/lib/api/model/info.ex @@ -1,5 +1,5 @@ -defmodule Scrapper.Api.Model.Info do - alias Scrapper.Api.Model.Participant +defmodule LoLAPI.Model.Info do + alias LoLAPI.Model.Participant defstruct endOfGameResult: "", gameCreation: "", diff --git a/apps/lol_api/lib/api/model/match_response.ex b/apps/lol_api/lib/api/model/match_response.ex new file mode 100644 index 0000000..f08f7c5 --- /dev/null +++ b/apps/lol_api/lib/api/model/match_response.ex @@ -0,0 +1,6 @@ +defmodule LoLAPI.Model.MatchResponse do + alias LoLAPI.Model.{Info, Metadata} + + defstruct metadata: %Metadata{}, + info: %Info{} +end diff --git a/apps/scrapper/lib/scrapper/api/model/metadata.ex b/apps/lol_api/lib/api/model/metadata.ex similarity index 57% rename from apps/scrapper/lib/scrapper/api/model/metadata.ex rename to apps/lol_api/lib/api/model/metadata.ex index 1dbfe00..289c0fd 100644 --- a/apps/scrapper/lib/scrapper/api/model/metadata.ex +++ b/apps/lol_api/lib/api/model/metadata.ex @@ -1,3 +1,3 @@ -defmodule Scrapper.Api.Model.Metadata do +defmodule LoLAPI.Model.Metadata do defstruct [:dataVersion, :matchId, :participants] end diff --git a/apps/scrapper/lib/scrapper/api/model/participant.ex b/apps/lol_api/lib/api/model/participant.ex similarity index 98% rename from apps/scrapper/lib/scrapper/api/model/participant.ex rename to apps/lol_api/lib/api/model/participant.ex index bd4ec9a..739f421 100644 --- a/apps/scrapper/lib/scrapper/api/model/participant.ex +++ b/apps/lol_api/lib/api/model/participant.ex @@ -1,4 +1,4 @@ -defmodule Scrapper.Api.Model.Participant do +defmodule LoLAPI.Model.Participant do # Enum.map(participant, fn {k,_v} -> ":#{k}" end) |> Enum.join(", ") defstruct [ :onMyWayPings, diff --git a/apps/lol_api/lib/lol_api.ex b/apps/lol_api/lib/lol_api.ex new file mode 100644 index 0000000..df62b56 --- /dev/null +++ b/apps/lol_api/lib/lol_api.ex @@ -0,0 +1,18 @@ +defmodule LoLAPI do + @moduledoc """ + Documentation for `LoLAPI`. + """ + + @doc """ + Hello world. + + ## Examples + + iex> LoLAPI.hello() + :world + + """ + def hello do + :world + end +end diff --git a/apps/lol_api/mix.exs b/apps/lol_api/mix.exs new file mode 100644 index 0000000..3e941b2 --- /dev/null +++ b/apps/lol_api/mix.exs @@ -0,0 +1,35 @@ +defmodule LoLAPI.MixProject do + use Mix.Project + + def project do + [ + app: :lol_api, + version: "0.1.0", + build_path: "../../_build", + config_path: "../../config/config.exs", + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.16", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications. + def application do + [ + extra_applications: [:logger] + ] + end + + # Run "mix help deps" to learn about dependencies. + defp deps do + [ + {:httpoison, "~> 2.2"}, + {:poison, "~> 5.0"} + # {:dep_from_hexpm, "~> 0.3.0"}, + # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}, + # {:sibling_app_in_umbrella, in_umbrella: true} + ] + end +end diff --git a/apps/lol_api/test/lo_lapi_test.exs b/apps/lol_api/test/lo_lapi_test.exs new file mode 100644 index 0000000..a2cbd4d --- /dev/null +++ b/apps/lol_api/test/lo_lapi_test.exs @@ -0,0 +1,8 @@ +defmodule LoLAPITest do + use ExUnit.Case + doctest LoLAPI + + test "greets the world" do + assert LoLAPI.hello() == :world + end +end diff --git a/apps/lol_api/test/test_helper.exs b/apps/lol_api/test/test_helper.exs new file mode 100644 index 0000000..869559e --- /dev/null +++ b/apps/lol_api/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() diff --git a/apps/scrapper/lib/scrapper/api/model/match_response.ex b/apps/scrapper/lib/scrapper/api/model/match_response.ex deleted file mode 100644 index da92b9d..0000000 --- a/apps/scrapper/lib/scrapper/api/model/match_response.ex +++ /dev/null @@ -1,6 +0,0 @@ -defmodule Scrapper.Api.Model.MatchResponse do - alias Scrapper.Api.Model.{Info, Metadata} - - defstruct metadata: %Metadata{}, - info: %Info{} -end diff --git a/apps/scrapper/lib/scrapper/match_classifier.ex b/apps/scrapper/lib/scrapper/match_classifier.ex new file mode 100644 index 0000000..b4b5b8b --- /dev/null +++ b/apps/scrapper/lib/scrapper/match_classifier.ex @@ -0,0 +1,31 @@ +defmodule Scrapper.MatchClassifier do + require Logger + + @spec classify_match(%LoLAPI.Model.MatchResponse{}) :: nil + def classify_match(match = %LoLAPI.Model.MatchResponse{}) do + classify_match_by_queue(match.info.queueId) + end + + @spec classify_match_by_queue(String.t()) :: nil + def classify_match_by_queue("420") do + matches = Storage.MatchStorage.S3MatchStorage.list_matches() + total_matches = Enum.count(matches) + + matches + |> Enum.with_index(fn match, index -> {match, index} end) + |> Scrapper.Parallel.peach(fn {match, index} -> + %{key: json_file} = match + [key | _] = String.split(json_file, ".") + Logger.info("Match at #{index} of #{total_matches} is classified") + response = HTTPoison.get!("http://localhost:9000/matches/#{key}.json", [], timeout: 5000) + %{"info" => %{"gameVersion" => gameVersion}} = Poison.decode!(response.body) + Storage.MatchStorage.S3MatchStorage.store_match(key, response.body, "ranked", gameVersion) + match + end) + end + + # pass functions, not data + + def classify_match_by_queue(_) do + end +end diff --git a/apps/scrapper/lib/scrapper/parallel/Parallel.ex b/apps/scrapper/lib/scrapper/parallel/Parallel.ex new file mode 100644 index 0000000..efb3fe9 --- /dev/null +++ b/apps/scrapper/lib/scrapper/parallel/Parallel.ex @@ -0,0 +1,7 @@ +defmodule Scrapper.Parallel do + def peach(enum, fun, concurrency \\ 10, timeout \\ :infinity) do + Task.async_stream(enum, &fun.(&1), max_concurrency: concurrency, timeout: timeout) + |> Stream.each(fn {:ok, val} -> val end) + |> Enum.to_list() + end +end diff --git a/apps/scrapper/lib/scrapper/processor/match_processor.ex b/apps/scrapper/lib/scrapper/processor/match_processor.ex index 25ba8ca..865db10 100644 --- a/apps/scrapper/lib/scrapper/processor/match_processor.ex +++ b/apps/scrapper/lib/scrapper/processor/match_processor.ex @@ -20,7 +20,7 @@ defmodule Scrapper.Processor.MatchProcessor do ]}, concurrency: 1, rate_limiting: [ - interval: 1000 * 1, + interval: 333 * 1, allowed_messages: 1 ] ], @@ -36,15 +36,15 @@ defmodule Scrapper.Processor.MatchProcessor do def handle_message(_, message = %Broadway.Message{}, _) do match_id = message.data - resp = Scrapper.Data.Api.MatchApi.get_match_by_id(match_id) + resp = LoLAPI.MatchApi.get_match_by_id(match_id) process_resp(resp, match_id) message end def process_resp({:ok, raw_match}, match_id) do - decoded_match = Poison.decode!(raw_match, as: %Scrapper.Api.Model.MatchResponse{}) - match_url = Scrapper.Storage.S3MatchStorage.store_match(match_id, raw_match) + decoded_match = Poison.decode!(raw_match, as: %LoLAPI.Model.MatchResponse{}) + match_url = Storage.MatchStorage.S3MatchStorage.store_match(match_id, raw_match) match = LolAnalytics.Match.MatchRepo.get_match(match_id) case match do @@ -59,6 +59,8 @@ defmodule Scrapper.Processor.MatchProcessor do end decoded_match.metadata.participants + |> Enum.shuffle() + |> Enum.take(2) |> Enum.each(fn participant_puuid -> Scrapper.Queue.PlayerQueue.queue_puuid(participant_puuid) end) diff --git a/apps/scrapper/lib/scrapper/processor/player_processor.ex b/apps/scrapper/lib/scrapper/processor/player_processor.ex index 9e4bffc..ac652e6 100644 --- a/apps/scrapper/lib/scrapper/processor/player_processor.ex +++ b/apps/scrapper/lib/scrapper/processor/player_processor.ex @@ -46,7 +46,7 @@ defmodule Scrapper.Processor.PlayerProcessor do update_player_processed(player) end - match_history = Scrapper.Data.Api.MatchApi.get_matches_from_player(puuid) + match_history = LoLAPI.MatchApi.get_matches_from_player(puuid) case match_history do {:ok, matches} -> diff --git a/apps/scrapper/lib/scrapper/storage/match_storage.ex b/apps/scrapper/lib/scrapper/storage/match_storage.ex deleted file mode 100644 index f48cc97..0000000 --- a/apps/scrapper/lib/scrapper/storage/match_storage.ex +++ /dev/null @@ -1,4 +0,0 @@ -defmodule Scrapper.Data.Storage.MatchStorage do - @callback get_match(String.t()) :: {:ok, Scrapper.Data.Match.t()} | {:error, :not_found} - @callback save_match(String.t(), Scrapper.Data.Match.t()) :: :ok -end diff --git a/apps/scrapper/mix.exs b/apps/scrapper/mix.exs index 1a88854..fabb519 100644 --- a/apps/scrapper/mix.exs +++ b/apps/scrapper/mix.exs @@ -26,15 +26,12 @@ defmodule Scrapper.MixProject do # Run "mix help deps" to learn about dependencies. defp deps do [ - {:httpoison, "~> 2.2"}, {:poison, "~> 5.0"}, - {:ex_aws, "~> 2.1"}, - {:ex_aws_s3, "~> 2.5.3"}, - {:hackney, "~> 1.9"}, - {:sweet_xml, "~> 0.6"}, {:broadway_rabbitmq, "~> 0.7"}, {:amqp, "~> 3.3"}, - {:lol_analytics, in_umbrella: true} + {:lol_analytics, in_umbrella: true}, + {:lol_api, in_umbrella: true}, + {:storage, in_umbrella: true} # {:dep_from_hexpm, "~> 0.3.0"}, # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}, # {:sibling_app_in_umbrella, in_umbrella: true} diff --git a/apps/storage/.formatter.exs b/apps/storage/.formatter.exs new file mode 100644 index 0000000..d2cda26 --- /dev/null +++ b/apps/storage/.formatter.exs @@ -0,0 +1,4 @@ +# Used by "mix format" +[ + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] +] diff --git a/apps/storage/.gitignore b/apps/storage/.gitignore new file mode 100644 index 0000000..f6d6326 --- /dev/null +++ b/apps/storage/.gitignore @@ -0,0 +1,26 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +storage-*.tar + +# Temporary files, for example, from tests. +/tmp/ diff --git a/apps/storage/README.md b/apps/storage/README.md new file mode 100644 index 0000000..93ad092 --- /dev/null +++ b/apps/storage/README.md @@ -0,0 +1,21 @@ +# Storage + +**TODO: Add description** + +## Installation + +If [available in Hex](https://hex.pm/docs/publish), the package can be installed +by adding `storage` to your list of dependencies in `mix.exs`: + +```elixir +def deps do + [ + {:storage, "~> 0.1.0"} + ] +end +``` + +Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc) +and published on [HexDocs](https://hexdocs.pm). Once published, the docs can +be found at . + diff --git a/apps/storage/config/config.exs b/apps/storage/config/config.exs new file mode 100644 index 0000000..d1186fe --- /dev/null +++ b/apps/storage/config/config.exs @@ -0,0 +1,3 @@ +import Config + +import_config "#{config_env()}.exs" diff --git a/apps/storage/config/dev.exs b/apps/storage/config/dev.exs new file mode 100644 index 0000000..423357e --- /dev/null +++ b/apps/storage/config/dev.exs @@ -0,0 +1,3 @@ +import Config + +import_config("libs/ex_aws_dev.exs") diff --git a/apps/storage/config/libs/ex_aws_dev.exs b/apps/storage/config/libs/ex_aws_dev.exs new file mode 100644 index 0000000..e0292c1 --- /dev/null +++ b/apps/storage/config/libs/ex_aws_dev.exs @@ -0,0 +1,17 @@ +import Config + +# config :ex_aws, :s3, +# scheme: "http://", +# host: System.get_env("EX_AWS_HOST"), +# port: System.get_env("EX_AWS_PORT"), +# access_key_id: System.get_env("EX_AWS_ACCESS_KEY"), +# secret_access_key: System.get_env("EX_AWS_SECRET_KEY") + +config :ex_aws, + access_key_id: "3zwMWl4RPCs8CHzhKmIX", + secret_access_key: "79B6LmryjJElkrIiHgDcfIxSpmvrLdVy75MyAJC2", + s3: [ + scheme: "http://", + host: "localhost", + port: "9000" + ] diff --git a/apps/storage/config/prod.exs b/apps/storage/config/prod.exs new file mode 100644 index 0000000..becde76 --- /dev/null +++ b/apps/storage/config/prod.exs @@ -0,0 +1 @@ +import Config diff --git a/apps/storage/config/test.exs b/apps/storage/config/test.exs new file mode 100644 index 0000000..e69de29 diff --git a/apps/storage/lib/match_storage/match_storage.ex b/apps/storage/lib/match_storage/match_storage.ex new file mode 100644 index 0000000..a2d9e74 --- /dev/null +++ b/apps/storage/lib/match_storage/match_storage.ex @@ -0,0 +1,6 @@ +defmodule Storage.MatchStorage do + @callback get_match(String.t()) :: {:ok, Scrapper.Data.Match.t()} | {:error, :not_found} + @callback save_match(String.t(), Scrapper.Data.Match.t()) :: :ok + @callback list_matches() :: [map()] + @callback store_match(match_id :: String.t(), match :: map(), path :: String.t()) :: String.t() +end diff --git a/apps/scrapper/lib/scrapper/storage/s3_match_storage.ex b/apps/storage/lib/match_storage/s3_match_storage.ex similarity index 61% rename from apps/scrapper/lib/scrapper/storage/s3_match_storage.ex rename to apps/storage/lib/match_storage/s3_match_storage.ex index 9d9555a..fddc929 100644 --- a/apps/scrapper/lib/scrapper/storage/s3_match_storage.ex +++ b/apps/storage/lib/match_storage/s3_match_storage.ex @@ -1,14 +1,13 @@ -defmodule Scrapper.Storage.S3MatchStorage do +defmodule Storage.MatchStorage.S3MatchStorage do require Logger - import SweetXml - - @behaviour Scrapper.Data.Storage.MatchStorage + @behaviour Storage.MatchStorage def get_match(match_id) do "" end # check for to get all pages next_continuation_token + @impl true def list_matches() do {:ok, %{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} = ExAws.S3.list_objects_v2("matches") @@ -24,7 +23,7 @@ defmodule Scrapper.Storage.S3MatchStorage do end @spec list_matches(list(String.t()), String.t()) :: list(String.t()) - def list_matches(acc, continuation_token) do + defp list_matches(acc, continuation_token) do resp = {:ok, %{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} = @@ -38,22 +37,38 @@ defmodule Scrapper.Storage.S3MatchStorage do end end - def download_match(destination_path, url) do - ExAws.S3.download_file(url, destination_path) - |> ExAws.request() - end - @doc """ - iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content" + iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content", "matches" """ - @spec store_match(String.t(), String.t()) :: none() - def store_match(match_id, match_data) do + @impl true + @spec store_match(String.t(), String.t(), String.t()) :: none() + def store_match(match_id, match_data, bucket) do File.write("/tmp/#{match_id}.json", match_data) url = "/tmp/#{match_id}.json" |> ExAws.S3.Upload.stream_file() - |> ExAws.S3.upload("matches", "#{match_id}.json") + |> ExAws.S3.upload(bucket, "#{match_id}.json") + |> ExAws.request!() + |> extract_s3_url_from_upload + + Logger.info("Stored match at #{url}") + + url + end + + @doc """ + iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content", "ranked" "14.9" + """ + @impl true + @spec store_match(String.t(), String.t(), String.t(), String.t()) :: none() + def store_match(match_id, match_data, bucket, path) do + File.write("/tmp/#{match_id}.json", match_data) + + url = + "/tmp/#{match_id}.json" + |> ExAws.S3.Upload.stream_file() + |> ExAws.S3.upload("#{bucket}/#{path}", "#{match_id}.json") |> ExAws.request!() |> extract_s3_url_from_upload diff --git a/apps/storage/lib/storage.ex b/apps/storage/lib/storage.ex new file mode 100644 index 0000000..5386638 --- /dev/null +++ b/apps/storage/lib/storage.ex @@ -0,0 +1,18 @@ +defmodule Storage do + @moduledoc """ + Documentation for `Storage`. + """ + + @doc """ + Hello world. + + ## Examples + + iex> Storage.hello() + :world + + """ + def hello do + :world + end +end diff --git a/apps/storage/mix.exs b/apps/storage/mix.exs new file mode 100644 index 0000000..ae7e50b --- /dev/null +++ b/apps/storage/mix.exs @@ -0,0 +1,37 @@ +defmodule Storage.MixProject do + use Mix.Project + + def project do + [ + app: :storage, + version: "0.1.0", + build_path: "../../_build", + config_path: "../../config/config.exs", + deps_path: "../../deps", + lockfile: "../../mix.lock", + elixir: "~> 1.16", + start_permanent: Mix.env() == :prod, + deps: deps() + ] + end + + # Run "mix help compile.app" to learn about applications. + def application do + [ + extra_applications: [:logger] + ] + end + + # Run "mix help deps" to learn about dependencies. + defp deps do + [ + {:ex_aws, "~> 2.1"}, + {:ex_aws_s3, "~> 2.5.3"}, + {:hackney, "~> 1.9"}, + {:sweet_xml, "~> 0.6"} + # {:dep_from_hexpm, "~> 0.3.0"}, + # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}, + # {:sibling_app_in_umbrella, in_umbrella: true} + ] + end +end diff --git a/apps/storage/test/storage_test.exs b/apps/storage/test/storage_test.exs new file mode 100644 index 0000000..40211c0 --- /dev/null +++ b/apps/storage/test/storage_test.exs @@ -0,0 +1,8 @@ +defmodule StorageTest do + use ExUnit.Case + doctest Storage + + test "greets the world" do + assert Storage.hello() == :world + end +end diff --git a/apps/storage/test/test_helper.exs b/apps/storage/test/test_helper.exs new file mode 100644 index 0000000..869559e --- /dev/null +++ b/apps/storage/test/test_helper.exs @@ -0,0 +1 @@ +ExUnit.start() diff --git a/config/config.exs b/config/config.exs index c165705..4c225a1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -66,3 +66,5 @@ config :phoenix, :json_library, Jason # of this file so it overrides the configuration defined above. import_config "#{config_env()}.exs" import_config "../apps/scrapper/config/config.exs" +import_config "../apps/storage/config/config.exs" +import_config "../apps/lol_api/config/config.exs"