move storage and api to apps, create base analyzer

.gitignore

@@ -24,4 +24,4 @@ erl_crash.dump
 
 .env
 
-
+.DS_store

apps/lol_analytics/lib/lol_analytics/analyzer/base_analyzer.ex

@@ -0,0 +1,3 @@
+defmodule LolAnalytics.Analyzer.BaseAnalyzer do
+  @callback analyze(:url, path :: String.t()) :: :ok
+end

apps/lol_analytics/lib/lol_analytics/analyzer/champion_analyzer.ex

@@ -0,0 +1,32 @@
+defmodule LolAnalytics.Analyzer.ChampionAnalyzer do
+  alias Hex.HTTP
+  @behaviour LolAnalytics.Analyzer.BaseAnalyzer
+
+  @doc """
+  iex> LolAnalytics.Analyzer.ChampionAnalyzer.analyze(:url, "https://na1.api.riotgames.com/lol/match/v4/match/234567890123456789")
+  :ok
+  """
+  @impl true
+  @spec analyze(atom(), String.t()) :: :ok
+  def analyze(:url, path) do
+    data = HTTPoison.get!(path)
+    analyze(:data, data.body)
+    :ok
+  end
+
+  @doc """
+  iex> LolAnalytics.Analyzer.ChampionAnalyzer.analyze(:url, "http://localhost:9000/ranked/14.9.580.2108/EUW1_6923309745.json")
+  """
+  @impl true
+  @spec analyze(atom(), any()) :: list(LoLAPI.Model.Participant.t())
+  def analyze(:data, data) do
+    decoded = Poison.decode!(data)
+
+    %{"info" => %{"participants" => participants}} = decoded
+
+    participants
+    |> Enum.each(fn %{"win" => win, "championId" => champion_id} ->
+      IO.inspect(%{win: win, champion_id: champion_id})
+    end)
+  end
+end

apps/lol_analytics/lib/lol_analytics/player/player_repo.ex

@@ -15,7 +15,9 @@ defmodule LolAnalytics.Player.PlayerRepo do
 
   def get_player(puuid) do
     query = from p in PlayerSchema, where: p.puuid == ^puuid
-    LoLAnalytics.Repo.one(query)
+
+    LoLAnalytics.Repo.all(query)
+    |> List.first()
   end
 
   @spec insert_player(String.t(), keyword()) :: %PlayerSchema{}

apps/lol_analytics/mix.exs

@@ -40,7 +40,10 @@ defmodule LoLAnalytics.MixProject do
       {:phoenix_pubsub, "~> 2.1"},
       {:ecto_sql, "~> 3.10"},
       {:postgrex, ">= 0.0.0"},
-      {:jason, "~> 1.2"}
+      {:jason, "~> 1.2"},
+      {:lol_api, in_umbrella: true},
+      {:httpoison, "~> 2.2"},
+      {:poison, "~> 5.0"}
     ]
   end
 

apps/lol_analytics/priv/repo/migrations/20240502203545_player.exs

@@ -10,6 +10,7 @@ defmodule LoLAnalytics.Repo.Migrations.Player do
       timestamps()
     end
 
+    unique_index("player", :puuid)
     create index(:player, [:puuid])
   end
 end

apps/lol_api/.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

apps/lol_api/.gitignore

@@ -0,0 +1,26 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+lol_api-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/

apps/lol_api/README.md

@@ -0,0 +1,21 @@
+# LoLAPI
+
+**TODO: Add description**
+
+## Installation
+
+If [available in Hex](https://hex.pm/docs/publish), the package can be installed
+by adding `lol_api` to your list of dependencies in `mix.exs`:
+
+```elixir
+def deps do
+  [
+    {:lol_api, "~> 0.1.0"}
+  ]
+end
+```
+
+Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
+and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
+be found at <https://hexdocs.pm/lol_api>.
+

apps/lol_api/config/config.exs

@@ -0,0 +1,4 @@
+import Config
+
+config :lol_api,
+  riot_api_key: System.get_env("RIOT_API_KEY")

apps/lol_api/lib/api/account_api.ex

@@ -1,4 +1,6 @@
-defmodule Scrapper.Api.AccountApi do
+defmodule LoLAPI.AccountApi do
+  require Logger
+
   @get_puuid_endpoint "https://europe.api.riotgames.com/riot/account/v1/accounts/by-riot-id/%{gameName}/%{tagLine}"
 
   @spec get_puuid(String.t(), String.t()) :: {:ok, String.t()} | {:error, String.t()}

apps/lol_api/lib/api/match_api.ex

@@ -1,14 +1,14 @@
-defmodule Scrapper.Data.Api.MatchApi do
+defmodule LoLAPI.MatchApi do
-  import Logger
+  require Logger
   @match_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/%{matchid}"
   @puuid_matches_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/by-puuid/%{puuid}/ids"
 
   @doc """
   Get match by id
 
-  iex> Scrapper.Data.MatchApi.get_match_by_id("EUW1_6921743825")
+  iex> LoLAPI.MatchApi.get_match_by_id("EUW1_6921743825")
   """
-  @spec get_match_by_id(String.t()) :: %Scrapper.Api.Model.MatchResponse{}
+  @spec get_match_by_id(String.t()) :: %LoLAPI.Model.MatchResponse{}
   def get_match_by_id(match_id) do
     url = String.replace(@match_base_endpoint, "%{matchid}", match_id)
     Logger.info("Making request to #{url}")
@@ -29,7 +29,7 @@ defmodule Scrapper.Data.Api.MatchApi do
   @doc """
   Get matches from player
 
-  iex> Scrapper.Data.Api.MatchApi.get_matches_from_player "JB6TdEWlKjZwnbgdSzOogYepNfjLPdUh68S8b4kUu4EEZy4R4MMAgv92QMj1XgVjtzHmZVLaOW7mzg"
+  iex> LoLAPI.MatchApi.get_matches_from_player "JB6TdEWlKjZwnbgdSzOogYepNfjLPdUh68S8b4kUu4EEZy4R4MMAgv92QMj1XgVjtzHmZVLaOW7mzg"
   """
   @spec get_matches_from_player(String.t()) :: list(String.t()) | integer()
   def get_matches_from_player(puuid) do

apps/lol_api/lib/api/model/info.ex

@@ -1,5 +1,5 @@
-defmodule Scrapper.Api.Model.Info do
+defmodule LoLAPI.Model.Info do
-  alias Scrapper.Api.Model.Participant
+  alias LoLAPI.Model.Participant
 
   defstruct endOfGameResult: "",
             gameCreation: "",

apps/lol_api/lib/api/model/match_response.ex

@@ -0,0 +1,6 @@
+defmodule LoLAPI.Model.MatchResponse do
+  alias LoLAPI.Model.{Info, Metadata}
+
+  defstruct metadata: %Metadata{},
+            info: %Info{}
+end

apps/lol_api/lib/api/model/metadata.ex

@@ -1,3 +1,3 @@
-defmodule Scrapper.Api.Model.Metadata do
+defmodule LoLAPI.Model.Metadata do
   defstruct [:dataVersion, :matchId, :participants]
 end

apps/lol_api/lib/api/model/participant.ex

@@ -1,4 +1,4 @@
-defmodule Scrapper.Api.Model.Participant do
+defmodule LoLAPI.Model.Participant do
   # Enum.map(participant, fn {k,_v} -> ":#{k}" end) |> Enum.join(", ")
   defstruct [
     :onMyWayPings,

apps/lol_api/lib/lol_api.ex

@@ -0,0 +1,18 @@
+defmodule LoLAPI do
+  @moduledoc """
+  Documentation for `LoLAPI`.
+  """
+
+  @doc """
+  Hello world.
+
+  ## Examples
+
+      iex> LoLAPI.hello()
+      :world
+
+  """
+  def hello do
+    :world
+  end
+end

apps/lol_api/mix.exs

@@ -0,0 +1,35 @@
+defmodule LoLAPI.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :lol_api,
+      version: "0.1.0",
+      build_path: "../../_build",
+      config_path: "../../config/config.exs",
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.16",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications.
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  # Run "mix help deps" to learn about dependencies.
+  defp deps do
+    [
+      {:httpoison, "~> 2.2"},
+      {:poison, "~> 5.0"}
+      # {:dep_from_hexpm, "~> 0.3.0"},
+      # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"},
+      # {:sibling_app_in_umbrella, in_umbrella: true}
+    ]
+  end
+end

apps/lol_api/test/lo_lapi_test.exs

@@ -0,0 +1,8 @@
+defmodule LoLAPITest do
+  use ExUnit.Case
+  doctest LoLAPI
+
+  test "greets the world" do
+    assert LoLAPI.hello() == :world
+  end
+end

apps/lol_api/test/test_helper.exs

@@ -0,0 +1 @@
+ExUnit.start()

apps/scrapper/lib/scrapper/api/model/match_response.ex

@@ -1,6 +0,0 @@
-defmodule Scrapper.Api.Model.MatchResponse do
-  alias Scrapper.Api.Model.{Info, Metadata}
-
-  defstruct metadata: %Metadata{},
-            info: %Info{}
-end

apps/scrapper/lib/scrapper/match_classifier.ex

@@ -0,0 +1,31 @@
+defmodule Scrapper.MatchClassifier do
+  require Logger
+
+  @spec classify_match(%LoLAPI.Model.MatchResponse{}) :: nil
+  def classify_match(match = %LoLAPI.Model.MatchResponse{}) do
+    classify_match_by_queue(match.info.queueId)
+  end
+
+  @spec classify_match_by_queue(String.t()) :: nil
+  def classify_match_by_queue("420") do
+    matches = Storage.MatchStorage.S3MatchStorage.list_matches()
+    total_matches = Enum.count(matches)
+
+    matches
+    |> Enum.with_index(fn match, index -> {match, index} end)
+    |> Scrapper.Parallel.peach(fn {match, index} ->
+      %{key: json_file} = match
+      [key | _] = String.split(json_file, ".")
+      Logger.info("Match at #{index} of #{total_matches} is classified")
+      response = HTTPoison.get!("http://localhost:9000/matches/#{key}.json", [], timeout: 5000)
+      %{"info" => %{"gameVersion" => gameVersion}} = Poison.decode!(response.body)
+      Storage.MatchStorage.S3MatchStorage.store_match(key, response.body, "ranked", gameVersion)
+      match
+    end)
+  end
+
+  # pass functions, not data
+
+  def classify_match_by_queue(_) do
+  end
+end

apps/scrapper/lib/scrapper/parallel/Parallel.ex

@@ -0,0 +1,7 @@
+defmodule Scrapper.Parallel do
+  def peach(enum, fun, concurrency \\ 10, timeout \\ :infinity) do
+    Task.async_stream(enum, &fun.(&1), max_concurrency: concurrency, timeout: timeout)
+    |> Stream.each(fn {:ok, val} -> val end)
+    |> Enum.to_list()
+  end
+end

apps/scrapper/lib/scrapper/processor/match_processor.ex

@@ -20,7 +20,7 @@ defmodule Scrapper.Processor.MatchProcessor do
            ]},
         concurrency: 1,
         rate_limiting: [
-          interval: 1000 * 1,
+          interval: 333 * 1,
           allowed_messages: 1
         ]
       ],
@@ -36,15 +36,15 @@ defmodule Scrapper.Processor.MatchProcessor do
   def handle_message(_, message = %Broadway.Message{}, _) do
     match_id = message.data
 
-    resp = Scrapper.Data.Api.MatchApi.get_match_by_id(match_id)
+    resp = LoLAPI.MatchApi.get_match_by_id(match_id)
     process_resp(resp, match_id)
 
     message
   end
 
   def process_resp({:ok, raw_match}, match_id) do
-    decoded_match = Poison.decode!(raw_match, as: %Scrapper.Api.Model.MatchResponse{})
+    decoded_match = Poison.decode!(raw_match, as: %LoLAPI.Model.MatchResponse{})
-    match_url = Scrapper.Storage.S3MatchStorage.store_match(match_id, raw_match)
+    match_url = Storage.MatchStorage.S3MatchStorage.store_match(match_id, raw_match)
     match = LolAnalytics.Match.MatchRepo.get_match(match_id)
 
     case match do
@@ -59,6 +59,8 @@ defmodule Scrapper.Processor.MatchProcessor do
     end
 
     decoded_match.metadata.participants
+    |> Enum.shuffle()
+    |> Enum.take(2)
     |> Enum.each(fn participant_puuid ->
       Scrapper.Queue.PlayerQueue.queue_puuid(participant_puuid)
     end)

apps/scrapper/lib/scrapper/processor/player_processor.ex

@@ -46,7 +46,7 @@ defmodule Scrapper.Processor.PlayerProcessor do
         update_player_processed(player)
     end
 
-    match_history = Scrapper.Data.Api.MatchApi.get_matches_from_player(puuid)
+    match_history = LoLAPI.MatchApi.get_matches_from_player(puuid)
 
     case match_history do
       {:ok, matches} ->

apps/scrapper/lib/scrapper/storage/match_storage.ex

@@ -1,4 +0,0 @@
-defmodule Scrapper.Data.Storage.MatchStorage do
-  @callback get_match(String.t()) :: {:ok, Scrapper.Data.Match.t()} | {:error, :not_found}
-  @callback save_match(String.t(), Scrapper.Data.Match.t()) :: :ok
-end

apps/scrapper/mix.exs

@@ -26,15 +26,12 @@ defmodule Scrapper.MixProject do
   # Run "mix help deps" to learn about dependencies.
   defp deps do
     [
-      {:httpoison, "~> 2.2"},
       {:poison, "~> 5.0"},
-      {:ex_aws, "~> 2.1"},
-      {:ex_aws_s3, "~> 2.5.3"},
-      {:hackney, "~> 1.9"},
-      {:sweet_xml, "~> 0.6"},
       {:broadway_rabbitmq, "~> 0.7"},
       {:amqp, "~> 3.3"},
-      {:lol_analytics, in_umbrella: true}
+      {:lol_analytics, in_umbrella: true},
+      {:lol_api, in_umbrella: true},
+      {:storage, in_umbrella: true}
       # {:dep_from_hexpm, "~> 0.3.0"},
       # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"},
       # {:sibling_app_in_umbrella, in_umbrella: true}

apps/storage/.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

apps/storage/.gitignore

@@ -0,0 +1,26 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+storage-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/

apps/storage/README.md

@@ -0,0 +1,21 @@
+# Storage
+
+**TODO: Add description**
+
+## Installation
+
+If [available in Hex](https://hex.pm/docs/publish), the package can be installed
+by adding `storage` to your list of dependencies in `mix.exs`:
+
+```elixir
+def deps do
+  [
+    {:storage, "~> 0.1.0"}
+  ]
+end
+```
+
+Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
+and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
+be found at <https://hexdocs.pm/storage>.
+

apps/storage/config/config.exs

@@ -0,0 +1,3 @@
+import Config
+
+import_config "#{config_env()}.exs"

apps/storage/config/dev.exs

@@ -0,0 +1,3 @@
+import Config
+
+import_config("libs/ex_aws_dev.exs")

apps/storage/config/libs/ex_aws_dev.exs

@@ -0,0 +1,17 @@
+import Config
+
+# config :ex_aws, :s3,
+#   scheme: "http://",
+#   host: System.get_env("EX_AWS_HOST"),
+#   port: System.get_env("EX_AWS_PORT"),
+#   access_key_id: System.get_env("EX_AWS_ACCESS_KEY"),
+#   secret_access_key: System.get_env("EX_AWS_SECRET_KEY")
+
+config :ex_aws,
+  access_key_id: "3zwMWl4RPCs8CHzhKmIX",
+  secret_access_key: "79B6LmryjJElkrIiHgDcfIxSpmvrLdVy75MyAJC2",
+  s3: [
+    scheme: "http://",
+    host: "localhost",
+    port: "9000"
+  ]

apps/storage/config/prod.exs

@@ -0,0 +1 @@
+import Config

apps/storage/lib/match_storage/match_storage.ex

@@ -0,0 +1,6 @@
+defmodule Storage.MatchStorage do
+  @callback get_match(String.t()) :: {:ok, Scrapper.Data.Match.t()} | {:error, :not_found}
+  @callback save_match(String.t(), Scrapper.Data.Match.t()) :: :ok
+  @callback list_matches() :: [map()]
+  @callback store_match(match_id :: String.t(), match :: map(), path :: String.t()) :: String.t()
+end

apps/storage/lib/match_storage/s3_match_storage.ex

@@ -1,14 +1,13 @@
-defmodule Scrapper.Storage.S3MatchStorage do
+defmodule Storage.MatchStorage.S3MatchStorage do
   require Logger
-  import SweetXml
+  @behaviour Storage.MatchStorage
-
-  @behaviour Scrapper.Data.Storage.MatchStorage
 
   def get_match(match_id) do
     ""
   end
 
   # check for to get all pages next_continuation_token
+  @impl true
   def list_matches() do
     {:ok, %{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} =
       ExAws.S3.list_objects_v2("matches")
@@ -24,7 +23,7 @@ defmodule Scrapper.Storage.S3MatchStorage do
   end
 
   @spec list_matches(list(String.t()), String.t()) :: list(String.t())
-  def list_matches(acc, continuation_token) do
+  defp list_matches(acc, continuation_token) do
     resp =
       {:ok,
        %{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} =
@@ -38,22 +37,38 @@ defmodule Scrapper.Storage.S3MatchStorage do
     end
   end
 
-  def download_match(destination_path, url) do
-    ExAws.S3.download_file(url, destination_path)
-    |> ExAws.request()
-  end
-
   @doc """
-  iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content"
+  iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content", "matches"
   """
-  @spec store_match(String.t(), String.t()) :: none()
+  @impl true
-  def store_match(match_id, match_data) do
+  @spec store_match(String.t(), String.t(), String.t()) :: none()
+  def store_match(match_id, match_data, bucket) do
     File.write("/tmp/#{match_id}.json", match_data)
 
     url =
       "/tmp/#{match_id}.json"
       |> ExAws.S3.Upload.stream_file()
-      |> ExAws.S3.upload("matches", "#{match_id}.json")
+      |> ExAws.S3.upload(bucket, "#{match_id}.json")
+      |> ExAws.request!()
+      |> extract_s3_url_from_upload
+
+    Logger.info("Stored match at #{url}")
+
+    url
+  end
+
+  @doc """
+  iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content", "ranked" "14.9"
+  """
+  @impl true
+  @spec store_match(String.t(), String.t(), String.t(), String.t()) :: none()
+  def store_match(match_id, match_data, bucket, path) do
+    File.write("/tmp/#{match_id}.json", match_data)
+
+    url =
+      "/tmp/#{match_id}.json"
+      |> ExAws.S3.Upload.stream_file()
+      |> ExAws.S3.upload("#{bucket}/#{path}", "#{match_id}.json")
       |> ExAws.request!()
       |> extract_s3_url_from_upload
 

apps/storage/lib/storage.ex

@@ -0,0 +1,18 @@
+defmodule Storage do
+  @moduledoc """
+  Documentation for `Storage`.
+  """
+
+  @doc """
+  Hello world.
+
+  ## Examples
+
+      iex> Storage.hello()
+      :world
+
+  """
+  def hello do
+    :world
+  end
+end

apps/storage/mix.exs

@@ -0,0 +1,37 @@
+defmodule Storage.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :storage,
+      version: "0.1.0",
+      build_path: "../../_build",
+      config_path: "../../config/config.exs",
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.16",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications.
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  # Run "mix help deps" to learn about dependencies.
+  defp deps do
+    [
+      {:ex_aws, "~> 2.1"},
+      {:ex_aws_s3, "~> 2.5.3"},
+      {:hackney, "~> 1.9"},
+      {:sweet_xml, "~> 0.6"}
+      # {:dep_from_hexpm, "~> 0.3.0"},
+      # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"},
+      # {:sibling_app_in_umbrella, in_umbrella: true}
+    ]
+  end
+end

apps/storage/test/storage_test.exs

@@ -0,0 +1,8 @@
+defmodule StorageTest do
+  use ExUnit.Case
+  doctest Storage
+
+  test "greets the world" do
+    assert Storage.hello() == :world
+  end
+end

apps/storage/test/test_helper.exs

@@ -0,0 +1 @@
+ExUnit.start()

config/config.exs

@@ -66,3 +66,5 @@ config :phoenix, :json_library, Jason
 # of this file so it overrides the configuration defined above.
 import_config "#{config_env()}.exs"
 import_config "../apps/scrapper/config/config.exs"
+import_config "../apps/storage/config/config.exs"
+import_config "../apps/lol_api/config/config.exs"