save display_mode to query string

README.md

@@ -1 +1,52 @@
-# LoLAnalytics.Umbrella
+# LoLAnalytics
+
+## Requirements
+
+- PostgreSQL
+- Elixir
+- RabbitMQ
+- MinIO
+
+A `docker-compose` file is provided to run them locally.
+
+## Environment variables
+
+The followign environment variables are required:
+
+```
+export RIOT_API_KEY="{API-KEY}"
+
+export EX_AWS_SECRET_KEY="{SECRET}"
+export EX_AWS_ACCESS_KEY="{ACCESS}"
+export EX_AWS_ENDPOINT="{HOST}"
+export EX_AWS_PORT="{PORT}" # minio defaults to 9000
+export DATABASE_URL="ecto://{USERNAME}:{PASSWORD}@{HOST}/lol-analytics"
+export SECRET_KEY_BASE="SECRET-KEY"
+```
+
+## Running
+
+```
+mix deps.get
+mix ecto.create && mix ecto.migrate
+iex -S mix phx.server
+```
+
+## Queueing a player
+
+```
+Scrapper.Queues.PlayerQueue.enqueue_puuid("PUUID")
+```
+
+The scrapper will retrieve player ranked history, enqueue it's teammates, and store every match in minio.
+
+## Processing
+
+A `Task` to process the matches stored for a patch can be spawned by running:
+```
+LolAnalytics.MatchesProcessor.process_for_patch "14.12.594.4901"
+```
+
+## Web site
+
+A web site is exposed at `localhost:4000`.

apps/lol_analytics/lib/lol_analytics/application.ex

@@ -11,7 +11,9 @@ defmodule LoLAnalytics.Application do
       LoLAnalytics.Repo,
       {DNSCluster, query: Application.get_env(:lol_analytics, :dns_cluster_query) || :ignore},
       {Phoenix.PubSub, name: LoLAnalytics.PubSub},
-      {Task.Supervisor, name: LoLAnalytics.TaskSupervisor}
+      {Task.Supervisor, name: LoLAnalytics.TaskSupervisor},
+      # {LolAnalytics.MatchProcessor.MatchesBroadwayProcessor, []},
+      {LolAnalytics.MatchProcessor.MatchesProducer, []}
       # Start a worker by calling: LoLAnalytics.Worker.start_link(arg)
       # {LoLAnalytics.Worker, arg}
     ]

apps/lol_analytics/lib/lol_analytics/champion_win_rate/champion_win_rate_repo.ex

@@ -1,56 +0,0 @@
-defmodule LolAnalytics.ChampionWinRate.ChampionWinRateRepo do
-  import Ecto.Query
-  alias LolAnalytics.ChampionWinRate.ChampionWinRateSchema
-  alias LoLAnalytics.Repo
-
-  @spec add_champion_win_rate(
-          champion_id :: String.t(),
-          patch :: String.t(),
-          position :: String.t(),
-          win? :: boolean
-        ) :: {:ok, ChampionWinRateSchema.t()} | {:error, Ecto.Changeset.t()}
-  def add_champion_win_rate(champion_id, patch, position, win?) do
-    Repo.transaction(fn ->
-      champion_query =
-        from cwr in LolAnalytics.ChampionWinRate.ChampionWinRateSchema,
-          where: cwr.champion_id == ^champion_id and cwr.position == ^position,
-          lock: "FOR UPDATE"
-
-      champion_data = Repo.one(champion_query)
-
-      case champion_data do
-        nil ->
-          ChampionWinRateSchema.changeset(%ChampionWinRateSchema{}, %{
-            champion_id: champion_id,
-            patch: patch,
-            total_games: 1,
-            position: position,
-            total_wins: if(win?, do: 1, else: 0)
-          })
-          |> Repo.insert!()
-
-        _ ->
-          total_games = champion_data.total_games + 1
-          total_wins = champion_data.total_wins + if win?, do: 1, else: 0
-
-          ChampionWinRateSchema.changeset(champion_data, %{
-            total_games: total_games,
-            total_wins: total_wins
-          })
-          |> Repo.update!()
-      end
-    end)
-  end
-
-  def list_win_rates() do
-    Repo.all(ChampionWinRateSchema)
-  end
-
-  def get_champion_win_rate(champion_id, _patch) do
-    champion_query =
-      from cwr in LolAnalytics.ChampionWinRate.ChampionWinRateSchema,
-        where: cwr.champion_id == ^champion_id
-
-    Repo.one(champion_query)
-  end
-end

apps/lol_analytics/lib/lol_analytics/champion_win_rate/champion_win_rate_schema.ex

@@ -1,20 +0,0 @@
-defmodule LolAnalytics.ChampionWinRate.ChampionWinRateSchema do
-  use Ecto.Schema
-  import Ecto.Changeset
-
-  schema "champion_win_rate" do
-    field :champion_id, :integer
-    field :total_games, :integer
-    field :patch, :string
-    field :position, :string
-    field :total_wins, :integer
-
-    timestamps()
-  end
-
-  def changeset(%__MODULE__{} = champion_win_rate, attrs) do
-    champion_win_rate
-    |> cast(attrs, [:champion_id, :total_games, :patch, :total_wins, :position])
-    |> validate_required([:champion_id, :total_games, :patch, :total_wins, :position])
-  end
-end

apps/lol_analytics/lib/lol_analytics/dimensions/item/item_metadata.ex

@@ -1,12 +1,9 @@
 defmodule LolAnalytics.Dimensions.Item.ItemMetadata do
   alias LolAnalytics.Dimensions.Item.ItemRepo
-  alias LolAnalytics.Dimensions.Champion.ChampionRepo
   @items_data_url "https://ddragon.leagueoflegends.com/cdn/14.11.1/data/en_US/item.json"
 
   def update_metadata() do
-    data = get_items()
+    get_items()
-
-    data
     |> Enum.each(&save_metadata/1)
   end
 

apps/lol_analytics/lib/lol_analytics/dimensions/item/item_schema.ex

@@ -13,6 +13,6 @@ defmodule LolAnalytics.Dimensions.Item.ItemSchema do
   def changeset(item = %__MODULE__{}, attrs \\ %{}) do
     item
     |> cast(attrs, @args)
-    |> validate_required(@args)
+    |> validate_required([:item_id])
   end
 end

apps/lol_analytics/lib/lol_analytics/dimensions/match/match_repo.ex

@@ -1,30 +1,85 @@
 defmodule LolAnalytics.Dimensions.Match.MatchRepo do
+  alias LolAnalytics.Dimensions.Patch.PatchRepo
   alias LolAnalytics.Dimensions.Match.MatchSchema
   alias LoLAnalytics.Repo
 
   import Ecto.Query
 
-  @spec get_or_create(String.t()) :: %MatchSchema{}
+  @spec get_or_create(%{
-  def get_or_create(match_id) do
+          :match_id => String.t(),
+          :queue_id => integer(),
+          :patch_number => String.t()
+        }) :: %MatchSchema{}
+  def get_or_create(%{match_id: match_id, queue_id: queue_id, patch_number: patch_number}) do
+    _patch = PatchRepo.get_or_create(patch_number)
     query = from m in MatchSchema, where: m.match_id == ^match_id
     match = Repo.one(query)
 
     case match do
       nil ->
-        match_changeset =
+        %MatchSchema{}
-          MatchSchema.changeset(
+        |> MatchSchema.changeset(%{
-            %MatchSchema{},
+          match_id: match_id,
-            %{match_id: match_id}
+          patch_number: patch_number,
-          )
+          queue_id: queue_id,
-
+          fact_champion_played_game_status: 0,
-        Repo.insert(match_changeset)
+          fact_champion_picked_item_status: 0,
+          fact_champion_picked_summoner_spell_status: 0
+        })
+        |> Repo.insert!()
 
       match ->
         match
     end
   end
 
+  @spec get(String.t()) :: nil | %MatchSchema{}
+  def get(match_id) do
+    query = from m in MatchSchema, where: m.match_id == ^match_id
+
+    Repo.one(query)
+  end
+
+  @type update_attrs :: %{
+          optional(:fact_champion_played_game_status) => process_status(),
+          optional(:fact_champion_picked_item_status) => process_status(),
+          optional(:fact_champion_picked_summoner_spell_status) => process_status()
+        }
+  @spec update(%MatchSchema{}, update_attrs()) :: %MatchSchema{}
+  def update(match, attrs) do
+    mapped_attrs =
+      attrs
+      |> Enum.map(fn {k, v} -> {k, process_status_atom_to_db(v)} end)
+      |> Map.new()
+
+    match
+    |> MatchSchema.changeset(mapped_attrs)
+    |> Repo.update!()
+  end
+
   def list_matches() do
     Repo.all(MatchSchema)
   end
+
+  def list_unprocessed_matches(limit, queue \\ 420) do
+    query =
+      from m in MatchSchema,
+        where:
+          (m.fact_champion_picked_item_status == 0 or
+             m.fact_champion_picked_summoner_spell_status == 0 or
+             m.fact_champion_played_game_status == 0) and
+            m.queue_id == ^queue,
+        order_by: [desc: m.updated_at],
+        limit: ^limit
+
+    Repo.all(query)
+  end
+
+  @type process_status :: :not_processed | :processed | :error
+  defp process_status_atom_to_db(:not_processed), do: 0
+  defp process_status_atom_to_db(:enqueued), do: 1
+  defp process_status_atom_to_db(:processed), do: 2
+  defp process_status_atom_to_db(:error), do: 3
+  defp process_status_atom_to_db(:error_match_not_found), do: 4
+  defp process_status_atom_to_db(_), do: raise("Invalid processing status")
 end

apps/lol_analytics/lib/lol_analytics/dimensions/match/match_schema.ex

@@ -2,14 +2,28 @@ defmodule LolAnalytics.Dimensions.Match.MatchSchema do
   use Ecto.Schema
   import Ecto.Changeset
 
+  @casting_attrs [
+    :match_id,
+    :queue_id,
+    :patch_number,
+    :fact_champion_picked_item_status,
+    :fact_champion_picked_summoner_spell_status,
+    :fact_champion_played_game_status
+  ]
+
   schema "dim_match" do
     field :match_id, :string
+    field :patch_number, :string
+    field :queue_id, :integer
+    field :fact_champion_picked_item_status, :integer
+    field :fact_champion_picked_summoner_spell_status, :integer
+    field :fact_champion_played_game_status, :integer
     timestamps()
   end
 
   def changeset(match = %__MODULE__{}, attrs \\ %{}) do
     match
-    |> cast(attrs, [:match_id])
+    |> cast(attrs, @casting_attrs)
     |> validate_required([:match_id])
   end
 end

apps/lol_analytics/lib/lol_analytics/dimensions/player/player_schema.ex

@@ -2,14 +2,20 @@ defmodule LolAnalytics.Dimensions.Player.PlayerSchema do
   use Ecto.Schema
   import Ecto.Changeset
 
+  @attrs [:puuid, :last_processed_at]
+
   schema "dim_player" do
     field :puuid, :string
+
+    field :last_processed_at, :utc_datetime,
+      default: DateTime.utc_now() |> DateTime.truncate(:second)
+
     timestamps()
   end
 
   def changeset(player = %__MODULE__{}, attrs \\ %{}) do
     player
-    |> cast(attrs, [:puuid])
+    |> cast(attrs, @attrs)
     |> validate_required([:puuid])
     |> unique_constraint([:puuid])
   end

apps/lol_analytics/lib/lol_analytics/facts/champion_picked_item/fact_processor.ex

@@ -1,7 +1,9 @@
 defmodule LolAnalytics.Facts.ChampionPickedItem.FactProcessor do
-  alias LolAnalytics.Facts.ChampionPickedItem.Repo
   require Logger
-  @behaviour LolAnalytics.Facts.FactBehaviour
+
+  alias LolAnalytics.Dimensions.Match.MatchSchema
+  alias LolAnalytics.Dimensions.Match.MatchRepo
+  alias LolAnalytics.Facts.ChampionPickedItem.Repo
 
   @doc """
 
@@ -20,10 +22,35 @@ defmodule LolAnalytics.Facts.ChampionPickedItem.FactProcessor do
     end
   end
 
+  @spec process_match(%MatchSchema{}) :: :ok | {:error, String.t()}
+  def process_match(match) do
+    match_url = "http://192.168.1.55:9000/ranked/#{match.patch_number}/#{match.match_id}.json"
+
+    with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
+           HTTPoison.get(match_url),
+         {:ok, decoded_match} <- Poison.decode(body, as: %LoLAPI.Model.MatchResponse{}) do
+      process_game_data(decoded_match)
+      MatchRepo.update(match, %{fact_champion_picked_item_status: :processed})
+      :ok
+    else
+      _ ->
+        MatchRepo.update(match, fact_champion_picked_item_status: :error_match_not_found)
+        Logger.error("Could not process data from #{match_url} for ChampionPickedItem")
+        {:error, "Could not process data from #{match_url}"}
+    end
+  end
+
   defp process_game_data(decoded_match) do
     participants = decoded_match.info.participants
     version = extract_game_version(decoded_match)
 
+    match =
+      MatchRepo.get_or_create(%{
+        match_id: decoded_match.metadata.matchId,
+        patch_number: decoded_match.info.gameVersion,
+        queue_id: decoded_match.info.queueId
+      })
+
     Logger.info("Processing ChampionPickedItem for match #{decoded_match.metadata.matchId}")
 
     participants

apps/lol_analytics/lib/lol_analytics/facts/champion_picked_item/repo.ex

@@ -31,7 +31,6 @@ defmodule LolAnalytics.Facts.ChampionPickedItem.Repo do
           :slot_number => integer()
         }) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()}
   def insert(attrs) do
-    _match = MatchRepo.get_or_create(attrs.match_id)
     _champion = ChampionRepo.get_or_create(attrs.champion_id)
     _player = PlayerRepo.get_or_create(attrs.puuid)
     _patch = PatchRepo.get_or_create(attrs.patch_number)

apps/lol_analytics/lib/lol_analytics/facts/champion_picked_summoner_spell/fact_processor.ex

@@ -1,21 +1,25 @@
 defmodule LolAnalytics.Facts.ChampionPickedSummonerSpell.FactProcessor do
-  @behaviour LolAnalytics.Facts.FactBehaviour
-
   require Logger
 
+  alias LolAnalytics.Dimensions.Match.MatchSchema
+  alias LolAnalytics.Dimensions.Match.MatchRepo
   alias LolAnalytics.Facts.ChampionPickedSummonerSpell
 
-  @impl true
+  @spec process_match(%MatchSchema{}) :: :ok | {:error, String.t()}
-  @spec process_game_at_url(String.t()) :: any()
+  def process_match(match) do
-  def process_game_at_url(url) do
+    match_url = "http://192.168.1.55:9000/ranked/#{match.patch_number}/#{match.match_id}.json"
+
     with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
-           HTTPoison.get(url),
+           HTTPoison.get(match_url),
          {:ok, decoded_match} <- Poison.decode(body, as: %LoLAPI.Model.MatchResponse{}) do
       process_game_data(decoded_match)
+      MatchRepo.update(match, %{fact_champion_picked_summoner_spell_status: :processed})
+      :ok
     else
       _ ->
-        Logger.error("Could not process data from #{url} for ChampionPickedSummonerSpell")
+        MatchRepo.update(match, fact_champion_picked_summoner_spell_status: :error_match_not_found)
-        {:error, "Could not process data from #{url}"}
+        Logger.error("Could not process data from #{match_url} for ChampionPickedItem")
+        {:error, "Could not process data from #{match_url}"}
     end
   end
 
@@ -23,6 +27,13 @@ defmodule LolAnalytics.Facts.ChampionPickedSummonerSpell.FactProcessor do
     participants = decoded_match.info.participants
     version = extract_game_version(decoded_match)
 
+    match =
+      MatchRepo.get_or_create(%{
+        match_id: decoded_match.metadata.matchId,
+        patch_number: decoded_match.info.gameVersion,
+        queue_id: decoded_match.info.queueId
+      })
+
     Logger.info("Processing ChampionPickedSummoner for match #{decoded_match.metadata.matchId}")
 
     participants

apps/lol_analytics/lib/lol_analytics/facts/champion_picked_summoner_spell/repo.ex

@@ -1,6 +1,7 @@
 defmodule LolAnalytics.Facts.ChampionPickedSummonerSpell.Repo do
   import Ecto.Query
 
+  alias LolAnalytics.Dimensions.Patch.PatchRepo
   alias LolAnalytics.Dimensions.SummonerSpell.SummonerSpellSchema
   alias LolAnalytics.Dimensions.SummonerSpell.SummonerSpellRepo
   alias LolAnalytics.Dimensions.Champion.ChampionSchema
@@ -13,19 +14,18 @@ defmodule LolAnalytics.Facts.ChampionPickedSummonerSpell.Repo do
 
   alias LoLAnalytics.Repo
 
-  @type insert_attrs :: %{
+  @spec insert(%{
-          match_id: String.t(),
+          :champion_id => String.t(),
-          champion_id: String.t(),
+          :match_id => String.t(),
-          puuid: String.t(),
+          :patch_number => String.t(),
-          summoner_spell_id: :integer
+          :puuid => any(),
-        }
+          :summoner_spell_id => String.t()
-
+        }) :: any()
-  @spec insert(insert_attrs()) :: any()
   def insert(attrs) do
-    _match = MatchRepo.get_or_create(attrs.match_id)
     _champion = ChampionRepo.get_or_create(attrs.champion_id)
     _player = PlayerRepo.get_or_create(attrs.puuid)
     _spell = SummonerSpellRepo.get_or_create(attrs.summoner_spell_id)
+    _patch = PatchRepo.get_or_create(attrs.patch_number)
 
     prev =
       from(f in Schema,
@@ -41,10 +41,14 @@ defmodule LolAnalytics.Facts.ChampionPickedSummonerSpell.Repo do
     |> Repo.insert_or_update()
   end
 
-  def get_champion_picked_summoners(champion_id, team_position \\ "MIDDLE") do
+  @spec get_champion_picked_summoners(String.t(), String.t(), String.t()) :: list()
+  def get_champion_picked_summoners(champion_id, team_position, patch_number) do
     query =
       from f in Schema,
-        where: f.champion_id == ^champion_id and f.team_position == ^team_position,
+        where:
+          f.champion_id == ^champion_id and
+            f.team_position == ^team_position and
+            f.patch_number == ^patch_number,
         join: c in ChampionSchema,
         on: c.champion_id == f.champion_id,
         join: s in SummonerSpellSchema,
@@ -74,8 +78,4 @@ defmodule LolAnalytics.Facts.ChampionPickedSummonerSpell.Repo do
 
     Repo.all(query)
   end
-
-  def list_facts() do
-    Repo.all(Schema)
-  end
 end

apps/lol_analytics/lib/lol_analytics/facts/champion_picked_summoner_spell/schema.ex

@@ -7,6 +7,7 @@ defmodule LolAnalytics.Facts.ChampionPickedSummonerSpell.Schema do
     :champion_id,
     :summoner_spell_id,
     :match_id,
+    :patch_number,
     :is_win,
     :game_length_seconds,
     :queue_id,
@@ -18,6 +19,7 @@ defmodule LolAnalytics.Facts.ChampionPickedSummonerSpell.Schema do
     field :champion_id, :integer
     field :summoner_spell_id, :integer
     field :match_id, :string
+    field :patch_number, :string
     field :is_win, :boolean
     field :game_length_seconds, :integer
     field :queue_id, :integer

apps/lol_analytics/lib/lol_analytics/facts/champion_played_game/champion_played_game_repo.ex

@@ -5,12 +5,10 @@ defmodule LolAnalytics.Facts.ChampionPlayedGame.Repo do
   alias LolAnalytics.Dimensions.Champion.ChampionSchema
   alias LolAnalytics.Dimensions.Player.PlayerRepo
   alias LolAnalytics.Dimensions.Champion.ChampionRepo
-  alias LolAnalytics.Dimensions.Match.MatchRepo
   alias LolAnalytics.Facts.ChampionPlayedGame.Schema
   alias LoLAnalytics.Repo
 
   def insert(attrs) do
-    _match = MatchRepo.get_or_create(attrs.match_id)
     _champion = ChampionRepo.get_or_create(attrs.champion_id)
     _player = PlayerRepo.get_or_create(attrs.puuid)
     _patch = PatchRepo.get_or_create(attrs.patch_number)

apps/lol_analytics/lib/lol_analytics/facts/champion_played_game/fact_processor.ex

@@ -1,19 +1,24 @@
 defmodule LolAnalytics.Facts.ChampionPlayedGame.FactProcessor do
   require Logger
 
-  @behaviour LolAnalytics.Facts.FactBehaviour
+  alias LolAnalytics.Dimensions.Match.MatchSchema
+  alias LolAnalytics.Dimensions.Match.MatchRepo
+
+  @spec process_match(%MatchSchema{}) :: :ok | {:error, String.t()}
+  def process_match(match) do
+    match_url = "http://192.168.1.55:9000/ranked/#{match.patch_number}/#{match.match_id}.json"
 
-  @impl true
-  @spec process_game_at_url(String.t()) :: none()
-  def process_game_at_url(url) do
     with {:ok, %HTTPoison.Response{status_code: 200, body: body}} <-
-           HTTPoison.get(url),
+           HTTPoison.get(match_url),
          {:ok, decoded_match} <- Poison.decode(body, as: %LoLAPI.Model.MatchResponse{}) do
       process_game_data(decoded_match)
+      MatchRepo.update(match, %{fact_champion_played_game_status: :processed})
+      :ok
     else
       _ ->
-        Logger.error("Could not process data from #{url} for ChampionPlayedGame")
+        MatchRepo.update(match, fact_champion_played_game_status: :error_match_not_found)
-        {:error, "Could not process data from #{url}"}
+        Logger.error("Could not process data from #{match_url} for ChampionPickedItem")
+        {:error, "Could not process data from #{match_url}"}
     end
   end
 
@@ -21,6 +26,13 @@ defmodule LolAnalytics.Facts.ChampionPlayedGame.FactProcessor do
     participants = decoded_match.info.participants
     version = extract_game_version(decoded_match)
 
+    match =
+      MatchRepo.get_or_create(%{
+        match_id: decoded_match.metadata.matchId,
+        patch_number: decoded_match.info.gameVersion,
+        queue_id: decoded_match.info.queueId
+      })
+
     Logger.info("Processing ChampionPlayedMatch for #{decoded_match.metadata.matchId}")
 
     participants

apps/lol_analytics/lib/lol_analytics/facts/fact_behaviour.ex

@@ -1,3 +0,0 @@
-defmodule LolAnalytics.Facts.FactBehaviour do
-  @callback process_game_at_url(String.t()) :: any()
-end

apps/lol_analytics/lib/lol_analytics/facts/facts_runner.ex

@@ -1,34 +1,18 @@
 defmodule LolAnalytics.Facts.FactsRunner do
   alias LolAnalytics.Facts
 
-  def analyze_by_patch(patch) do
+  def analyze_match(match) do
-    Storage.MatchStorage.S3MatchStorage.stream_files("ranked", patch: patch)
+    get_facts()
-    |> peach(fn %{key: path} ->
+    |> Enum.each(fn fact_runner ->
-      get_facts()
+      apply(fact_runner, [match])
-      |> Enum.each(fn fact_runner ->
-        apply(fact_runner, ["http://192.168.1.55:9000/ranked/#{path}"])
-      end)
     end)
   end
 
-  def analyze_all_matches do
-    Storage.MatchStorage.S3MatchStorage.stream_files("ranked")
-    |> peach(fn %{key: path} ->
-      get_facts()
-      |> Enum.each(fn fact_runner ->
-        apply(fact_runner, ["http://192.168.1.55:9000/ranked/#{path}"])
-      end)
-    end)
-  end
-
-  def analyze_match() do
-  end
-
   def get_facts() do
     [
-      &Facts.ChampionPickedSummonerSpell.FactProcessor.process_game_at_url/1,
+      &Facts.ChampionPickedSummonerSpell.FactProcessor.process_match/1,
-      &Facts.ChampionPlayedGame.FactProcessor.process_game_at_url/1,
+      &Facts.ChampionPlayedGame.FactProcessor.process_match/1,
-      &Facts.ChampionPickedItem.FactProcessor.process_game_at_url/1
+      &Facts.ChampionPickedItem.FactProcessor.process_match/1
     ]
   end
 

apps/lol_analytics/lib/lol_analytics/match/match_repo.ex

@@ -1,35 +0,0 @@
-defmodule LolAnalytics.Match.MatchRepo do
-  alias LolAnalytics.Match.MatchSchema
-
-  import Ecto.Query
-
-  def list_matches do
-    query = from m in MatchSchema, order_by: [desc: m.match_id]
-    LoLAnalytics.Repo.all(query)
-  end
-
-  def number_of_matches do
-    query = from m in MatchSchema, select: count(m.match_id)
-    LoLAnalytics.Repo.one(query)
-  end
-
-  @spec get_match(String.t()) :: %LolAnalytics.Match.MatchSchema{} | nil
-  def get_match(match_id) do
-    query = from m in MatchSchema, where: m.match_id == ^match_id
-
-    LoLAnalytics.Repo.one(query)
-  end
-
-  @spec insert_match(String.t()) :: %LolAnalytics.Match.MatchSchema{}
-  def insert_match(match_id) do
-    MatchSchema.changeset(%MatchSchema{}, %{:match_id => match_id, :processed => false})
-    |> LoLAnalytics.Repo.insert()
-  end
-
-  @spec update_match(%LolAnalytics.Match.MatchSchema{}, term()) ::
-          %LolAnalytics.Match.MatchSchema{}
-  def update_match(match, attrs) do
-    match = MatchSchema.changeset(match, attrs)
-    LoLAnalytics.Repo.update(match)
-  end
-end

apps/lol_analytics/lib/lol_analytics/match/match_schema.ex

@@ -1,19 +0,0 @@
-defmodule LolAnalytics.Match.MatchSchema do
-  use Ecto.Schema
-  import Ecto.Changeset
-
-  schema "match" do
-    field :match_id, :string
-    field :processed, :boolean, default: false
-    field :match_url, :string
-
-    timestamps()
-  end
-
-  def changeset(%__MODULE__{} = match, params \\ %{}) do
-    match
-    |> cast(params, [:match_id, :processed, :match_url])
-    |> validate_required([:match_id, :processed])
-  end
-
-end

apps/lol_analytics/lib/lol_analytics/match_processor/matches_broadway_processor.ex

@@ -0,0 +1,34 @@
+defmodule LolAnalytics.MatchProcessor.MatchesBroadwayProcessor do
+  alias LolAnalytics.Facts.FactsRunner
+  use Broadway
+
+  def start_link(opts) do
+    Broadway.start_link(__MODULE__,
+      name: __MODULE__,
+      processors: [default: []],
+      producer: [
+        module: {LolAnalytics.MatchProcessor.MatchesProducer, []},
+        rate_limiting: [
+          interval: 1000,
+          allowed_messages: 40
+        ]
+      ]
+    )
+  end
+
+  @impl Broadway
+  def handle_message(_processor, message, _context) do
+    message.data
+    # build_match_url(message.data.queue_id, message.data.patch_number, message.data.match_id)
+    |> FactsRunner.analyze_match()
+
+    message
+  end
+
+  defp build_match_url(queue, patch_id, match_id) do
+    "http://192.168.1.55:9000/#{queue_to_dir(queue)}/#{patch_id}/#{match_id}.json"
+  end
+
+  defp queue_to_dir(420), do: "ranked"
+  defp queue_to_dir(_), do: "ranked"
+end

apps/lol_analytics/lib/lol_analytics/match_processor/matches_producer.ex

@@ -0,0 +1,33 @@
+defmodule LolAnalytics.MatchProcessor.MatchesProducer do
+  use GenStage
+
+  @impl GenStage
+  def init(opts) do
+    {:producer, opts}
+  end
+
+  def start_link(opts) do
+    GenStage.start_link(__MODULE__, :ok)
+  end
+
+  @impl GenStage
+  def handle_demand(demand, state) do
+    matches = query_unprocessed_matches(demand)
+
+    {:noreply, matches, state}
+  end
+
+  defp query_unprocessed_matches(demand) when demand <= 0, do: []
+
+  defp query_unprocessed_matches(demand) do
+    LolAnalytics.Dimensions.Match.MatchRepo.list_unprocessed_matches(demand)
+    |> Enum.map(&broadway_transform/1)
+  end
+
+  defp broadway_transform(match) do
+    %Broadway.Message{
+      data: match,
+      acknowledger: Broadway.NoopAcknowledger.init()
+    }
+  end
+end

apps/lol_analytics/lib/lol_analytics/matches_processor.ex

@@ -1,12 +0,0 @@
-defmodule LolAnalytics.MatchesProcessor do
-  use GenServer
-
-  @doc """
-  iex> LolAnalytics.MatchesProcessor.process_for_patch "14.12.593.5894"
-  """
-  def process_for_patch(patch) do
-    Task.Supervisor.async(LoLAnalytics.TaskSupervisor, fn ->
-      LolAnalytics.Facts.FactsRunner.analyze_by_patch(patch)
-    end)
-  end
-end

apps/lol_analytics/mix.exs

@@ -44,7 +44,9 @@ defmodule LoLAnalytics.MixProject do
       {:lol_api, in_umbrella: true},
       {:storage, in_umbrella: true},
       {:httpoison, "~> 2.2"},
-      {:poison, "~> 5.0"}
+      {:poison, "~> 5.0"},
+      {:gen_stage, "~> 1.2.1"},
+      {:broadway, "~> 1.1"}
     ]
   end
 

apps/lol_analytics/priv/repo/migrations/20240526160430_analytics_tables.exs

@@ -16,13 +16,6 @@ defmodule LoLAnalytics.Repo.Migrations.AnalyticsTables do
 
     create index("dim_item", [:item_id], unique: true)
 
-    create table("dim_match") do
-      add :match_id, :string, primary_key: true, null: false
-      timestamps()
-    end
-
-    create index("dim_match", [:match_id], unique: true)
-
     create table("dim_patch") do
       add :patch_number, :string, primary_key: true, null: false
       timestamps()
@@ -30,6 +23,15 @@ defmodule LoLAnalytics.Repo.Migrations.AnalyticsTables do
 
     create index("dim_patch", [:patch_number], unique: true)
 
+    create table("dim_match") do
+      add :patch_number, references("dim_patch", column: :patch_number, type: :string)
+      add :match_id, :string, primary_key: true, null: false
+      timestamps()
+    end
+
+    create index("dim_match", [:patch_number])
+    create index("dim_match", [:match_id], unique: true)
+
     create table("dim_player") do
       add :puuid, :string, primary_key: true, null: false
       timestamps()

apps/lol_analytics/priv/repo/migrations/20240622103506_dim_match_fact_processing_status.exs

@@ -0,0 +1,15 @@
+defmodule LoLAnalytics.Repo.Migrations.DimMatchFactProcessingStatus do
+  use Ecto.Migration
+
+  def change do
+    alter table("dim_match") do
+      add :fact_champion_played_game_status, :integer, default: 0
+      add :fact_champion_picked_item_status, :integer, default: 0
+      add :fact_champion_picked_summoner_spell_status, :integer, default: 0
+    end
+
+    create index("dim_match", [:fact_champion_played_game_status])
+    create index("dim_match", [:fact_champion_picked_item_status])
+    create index("dim_match", [:fact_champion_picked_summoner_spell_status])
+  end
+end

apps/lol_analytics/priv/repo/migrations/20240622161637_dim_match_queue_id.exs

@@ -0,0 +1,9 @@
+defmodule LoLAnalytics.Repo.Migrations.DimMatchQueueId do
+  use Ecto.Migration
+
+  def change do
+    alter table "dim_match" do
+      add :queue_id, :integer
+    end
+  end
+end

apps/lol_analytics/priv/repo/migrations/20240624165516_dim_player_last_processed.exs

@@ -0,0 +1,9 @@
+defmodule LoLAnalytics.Repo.Migrations.DimPlayerLastProcessed do
+  use Ecto.Migration
+
+  def change do
+    alter table "dim_player" do
+      add :last_processed_at, :utc_datetime
+    end
+  end
+end

apps/lol_analytics_web/assets/js/hooks/champion_win_rate_patch.js

@@ -3,15 +3,24 @@ import Chart from "chart.js/auto"
 const ChampionWinRate = {
     mounted() {
         this.handleEvent("win-rate", ({ winRates }) => {
-            patches = winRates.map((winRate) => {
+            this.sortedWinRates = winRates.sort((a, b) => {
+                let [p1Major, p1Minor] = a.patch_number.split(".").map(Number);
+                let [p2Major, p2Minor] = b.patch_number.split(".").map(Number);
+
+
+                if (p1Major > p2Major || (p1Major == p2Major && p1Minor > p2Minor)) return 1;
+                return -1
+            })
+            this.patches = this.sortedWinRates.map((winRate) => {
                 return winRate.patch_number
             })
-            winRateValues = winRates.map((winRate) => winRate.win_rate)
+            this.winRateValues = this.sortedWinRates.map((winRate) => winRate.win_rate)
+            // TODO: it breaks on liveview updates, should apply a better fix...
             setInterval(() => {
                 const data = {
-                    labels: patches,
+                    labels: this.patches,
                     datasets: [{
-                        data: winRateValues,
+                        data: this.winRateValues,
                         fill: false,
                         borderColor: 'rgb(75, 192, 192)',
                         tension: 0.1

apps/lol_analytics_web/lib/lol_analytics_web/components/champion_components/champion_avatar.ex

@@ -16,7 +16,7 @@ defmodule LolAnalyticsWeb.ChampionComponents.ChampionAvatar do
       }
     </style>
     <div class="flex flex-col w-40">
-      <img src={@image} class="champion-avatar" alt="champion-icon" />
+      <img src={@image} class="champion-avatar rounded-xl" alt="champion-icon" />
     </div>
     """
   end

apps/lol_analytics_web/lib/lol_analytics_web/components/champion_components/items.ex

@@ -9,7 +9,7 @@ defmodule LolAnalyticsWeb.ChampionComponents.Items do
       <%= for item <- assigns.items do %>
         <div class="has-tooltip">
           <div clas="flex flex-col gap-1 p-4">
-            <img src={item.image} />
+            <img class="rounded-xl" src={item.image} />
             <%!-- <p><%= item.name %></p> --%>
             <p><%= item.win_rate %>%</p>
             <p class="text-xs"><%= item.wins %>/<%= item.total_games %></p>

apps/lol_analytics_web/lib/lol_analytics_web/components/champion_components/summoner_spells.ex

@@ -8,7 +8,7 @@ defmodule LolAnalyticsWeb.ChampionComponents.SummonerSpells do
       <%= for spell <- assigns.spells do %>
         <div class="has-tooltip">
           <div clas="flex flex-col gap-1">
-            <img src={spell.image} />
+            <img class="rounded-xl" src={spell.image} />
             <p><%= spell.win_rate %>%</p>
             <p class="text-xs"><%= spell.wins %>/<%= spell.total_games %></p>
           </div>

apps/lol_analytics_web/lib/lol_analytics_web/components/layouts/app.html.heex

@@ -4,9 +4,6 @@
       <a href="/">
         LoL Analytics
       </a>
-      <p class="bg-brand/5 text-brand rounded-full px-2 font-medium leading-6">
-        v<%= Application.spec(:phoenix, :vsn) %>
-      </p>
     </div>
     <div class="flex items-center gap-4 font-semibold leading-6 text-zinc-900">
       <.link patch="/champions">

apps/lol_analytics_web/lib/lol_analytics_web/components/patch_selector.ex

@@ -4,11 +4,16 @@ defmodule LolAnalyticsWeb.PatchSelector do
   def mount(socket) do
     patches =
       LolAnalytics.Dimensions.Patch.PatchRepo.list_patches()
+      |> Enum.map(fn patch ->
+        %{patch_number: String.split(patch.patch_number, ".") |> Enum.take(2) |> Enum.join(".")}
+      end)
+      |> MapSet.new()
+      |> Enum.to_list()
       |> Enum.sort(fn %{patch_number: p1}, %{patch_number: p2} ->
-        [_, minor_1] = String.split(p1, ".") |> Enum.map(&String.to_integer/1)
+        [major_1, minor_1] = String.split(p1, ".") |> Enum.map(&String.to_integer/1)
-        [_, minor_2] = String.split(p2, ".") |> Enum.map(&String.to_integer/1)
+        [major_2, minor_2] = String.split(p2, ".") |> Enum.map(&String.to_integer/1)
 
-        p1 > p2 && minor_1 > minor_2
+        major_1 > major_2 || (major_1 == major_2 && minor_1 > minor_2)
       end)
 
     patch_numbers = Enum.map(patches, & &1.patch_number)
@@ -33,6 +38,11 @@ defmodule LolAnalyticsWeb.PatchSelector do
   def render(assigns) do
     ~H"""
     <div phx-feedback-for={@id}>
+      <style>
+        .patch-selector {
+          width: 100px;
+        }
+      </style>
       <.form for={@patch_form} phx-change="validate" phx-target={@myself} phx-submit="save">
         <div class="flex gap-4">
           <p class="my-auto">Patch</p>
@@ -40,7 +50,7 @@ defmodule LolAnalyticsWeb.PatchSelector do
             phx-change="selected_patch"
             id="patch"
             name="patch"
-            class="block w-full rounded-md border border-gray-300 bg-white shadow-sm focus:border-zinc-400 focus:ring-0 sm:text-sm"
+            class="patch-selector cursor-pointer block w-full rounded-md border border-gray-300 bg-white shadow-sm focus:border-zinc-400 focus:ring-0 sm:text-sm"
           >
             <%= for patch <- @patch_numbers do %>
               <option key={patch} phx-click="select-patch" name={patch} value={patch}>

apps/lol_analytics_web/lib/lol_analytics_web/live/champion_live/components/champion_filters.ex

@@ -22,7 +22,6 @@ defmodule LolAnalyticsWeb.ChampionLive.Components.ChampionFilters do
   end
 
   attr :selectedrole, :string, required: true
-  attr :roles, :list, default: []
 
   def render(assigns) do
     selected_class =

apps/lol_analytics_web/lib/lol_analytics_web/live/champion_live/components/champions_list.ex

@@ -0,0 +1,79 @@
+defmodule LolAnalyticsWeb.ChampionLive.Components.ChampionsList do
+  use Phoenix.Component
+
+  attr :champion, :map
+
+  defp render_champion(assigns) do
+    detail_url =
+      "/champions/#{assigns.champion.id}?team-position=#{assigns.champion.team_position}&patch=#{assigns.champion.patch_number}"
+
+    ~H"""
+    <tr>
+      <td>
+        <.link patch={detail_url}>
+          <div class="flex cursor-pointer items-center gap-2">
+            <img
+              class="champion_image  rounded-md"
+              src={"https://ddragon.leagueoflegends.com/cdn/14.11.1/img/champion/#{assigns.champion.image}"}
+            />
+            <%= @champion.name %>
+          </div>
+        </.link>
+      </td>
+      <td><%= @champion.win_rate %>%</td>
+      <td><%= @champion.total_games %></td>
+      <td>
+        <img src={team_position_image(@champion.team_position)} class="w-5 h-5" />
+      </td>
+    </tr>
+    """
+  end
+
+  attr :champions, :list
+  attr :patch_number, :integer
+  attr :position, :string
+
+  def champions_list(assigns) do
+    ~H"""
+    <style>
+      .champion_image {
+        width: 50px;
+      }
+    </style>
+    <table class="table w-full">
+      <thead>
+        <tr>
+          <th scope="col" class="py-3 text-left text-gray-500 uppercase tracking-wider">
+            Champion
+          </th>
+          <th scope="col" class="py-3 text-left text-gray-500 uppercase tracking-wider">
+            Win rate
+          </th>
+          <th scope="col" class="py-3 text-left text-gray-500 uppercase tracking-wider">
+            Total games
+          </th>
+          <th scope="col" class="py-3 text-left text-gray-500 uppercase tracking-wider">
+            Position
+          </th>
+        </tr>
+      </thead>
+      <tbody>
+        <%= for champion <- assigns.champions do %>
+          <div class="cursor-pointer">
+            <.link patch={"/champions/#{champion.id}?team-position=#{champion.team_position}&patch=#{champion.patch_number}"}>
+              <.render_champion champion={champion} ) />
+            </.link>
+          </div>
+        <% end %>
+        <!-- table rows and cells go here -->
+      </tbody>
+    </table>
+    """
+  end
+
+  defp team_position_image("BOTTOM"), do: "/images/lanes/bot.png"
+  defp team_position_image("MIDDLE"), do: "/images/lanes/mid.png"
+  defp team_position_image("TOP"), do: "/images/lanes/top.png"
+  defp team_position_image("JUNGLE"), do: "/images/lanes/jungle.png"
+  defp team_position_image("UTILITY"), do: "/images/lanes/utility.png"
+end

apps/lol_analytics_web/lib/lol_analytics_web/live/champion_live/index.ex

@@ -2,6 +2,7 @@ defmodule LoLAnalyticsWeb.ChampionLive.Index do
   use LoLAnalyticsWeb, :live_view
 
   import LolAnalyticsWeb.ChampionComponents.ChampionCard
+  import LolAnalyticsWeb.ChampionLive.Components.ChampionsList
   import LolAnalyticsWeb.Loader
   import Phoenix.VerifiedRoutes
 
@@ -15,12 +16,14 @@ defmodule LoLAnalyticsWeb.ChampionLive.Index do
   def mount(params, _session, socket) do
     role = params["role"] || "all"
     patch = params["patch"] || "14.12"
+    display_mode = params["display_mode"] || "grid"
 
     socket =
       socket
       |> assign(:selected_role, role)
       |> assign(:selected_patch, patch)
       |> assign(:champions, %{status: :loading})
+      |> assign(:display_mode, display_mode)
       |> load_champs(role, patch)
 
     {:ok, socket}
@@ -28,18 +31,22 @@ defmodule LoLAnalyticsWeb.ChampionLive.Index do
 
   def handle_params(params, _uri, socket) do
     role = params["role"] || "all"
-    patch = params["patch"] || "14.12"
+    patch = params["patch"] || "14.10"
+    display_mode = params["display_mode"] || "grid"
 
     {
       :noreply,
       assign(socket, selected_role: role)
       |> assign(selected_patch: patch)
+      |> assign(display_mode: display_mode)
     }
   end
 
   @impl true
   def handle_event("filter", %{"role" => selected_role} = params, socket) do
-    query = Map.merge(params, %{patch: socket.assigns.selected_patch || "14.12"})
+    query =
+      get_query_params(socket)
+      |> Map.merge(%{role: selected_role})
 
     {:reply, %{},
      socket
@@ -49,9 +56,22 @@ defmodule LoLAnalyticsWeb.ChampionLive.Index do
      |> assign(:selected_role, selected_role)}
   end
 
+  def handle_event("set-display-mode", %{"mode" => mode} = params, socket) do
+    query_params =
+      get_query_params(socket)
+      |> Map.merge(%{display_mode: mode})
+
+    {:noreply,
+     assign(socket, :display_mode, mode)
+     |> push_patch(to: ~p"/champions?#{query_params}")}
+  end
+
   def handle_info(%{patch: patch}, socket) do
     selected_role = socket.assigns.selected_role
-    query_params = %{role: selected_role, patch: patch}
+
+    query_params =
+      get_query_params(socket)
+      |> Map.merge(%{patch: patch})
 
     socket =
       assign(socket, :champions, %{status: :loading})
@@ -93,13 +113,54 @@ defmodule LoLAnalyticsWeb.ChampionLive.Index do
     {:noreply, assign(socket, :champions, %{status: :data, data: champs})}
   end
 
-  def render_champions(assigns) do
+  def render_display_mode_selector_selector(assigns) do
+    ~H"""
+    <div class="flex">
+      <div phx-click="set-display-mode" phx-value-mode="grid" class="cursor-pointer">
+        <.icon name={grid_icon(assigns.display_mode)} alt="table" />
+      </div>
+      <div phx-click="set-display-mode" phx-value-mode="list" class="cursor-pointer">
+        <.icon name={list_icon(assigns.display_mode)} alt="table" />
+      </div>
+    </div>
+    """
+  end
+
+  defp grid_icon(selected_display_mode) do
+    case selected_display_mode do
+      "grid" -> "hero-squares-2x2-solid"
+      "list" -> "hero-squares-2x2"
+    end
+  end
+
+  def list_icon(selected_display_mode) do
+    case selected_display_mode do
+      "grid" -> "hero-table-cells"
+      "list" -> "hero-table-cells-solid"
+    end
+  end
+
+  def render_champions_list(assigns) do
     case assigns.champions do
       %{status: :loading} ->
         ~H"""
         <.loader />
         """
 
+      %{status: :data, data: champions} ->
+        ~H"""
+        <.champions_list champions={champions} />
+        """
+    end
+  end
+
+  def render_champions_grid(%{:champions => champions_state} = assigns) do
+    case champions_state do
+      %{status: :loading} ->
+        ~H"""
+        <.loader />
+        """
+
       %{status: :data, data: champions} ->
         ~H"""
         <div id="champions" class="grid grid-cols-2 sm:grid-cols-4  gap-4">
@@ -126,4 +187,12 @@ defmodule LoLAnalyticsWeb.ChampionLive.Index do
     socket
     |> assign(:page_title, "Listing Champions")
   end
+
+  defp get_query_params(socket) do
+    %{
+      patch: socket.assigns.selected_patch,
+      role: socket.assigns.selected_role,
+      display_mode: socket.assigns.display_mode
+    }
+  end
 end

apps/lol_analytics_web/lib/lol_analytics_web/live/champion_live/index.html.heex

@@ -1,7 +1,12 @@
 <.header>
-  Champions
+  <p class="text-3xl">
+    Champions
+  </p>
+
 </.header>
 
+<div class="h-4" />
+
 <div class="px-2">
   <h1 class="text-l font-semibold">Filters</h1>
 
@@ -10,9 +15,18 @@
     <.live_component module={ChampionFilters} id="role-filters" selectedrole={@selected_role || "all" } />
   </div>
 
-  <.live_component module={PatchSelector} id="patch-selector" />
+  <div class="flex justify-between items-center">
+    <.live_component module={PatchSelector} id="patch-selector" initial_patch={@selected_patch} />
+
+    <.render_display_mode_selector_selector display_mode={@display_mode} />
+  </div>
+
 
   <div class="h-4"></div>
 
-  <.render_champions champions={@champions} />
+  <%= if @display_mode=="grid" do %>
+    <.render_champions_grid champions={@champions} />
+    <% else %>
+      <.render_champions_list id="champions-list" champions={@champions} />
+      <% end %>
 </div>

apps/lol_analytics_web/lib/lol_analytics_web/live/champion_live/show.ex

@@ -20,7 +20,7 @@ defmodule LoLAnalyticsWeb.ChampionLive.Show do
      |> assign(:page_title, page_title(socket.assigns.live_action))
      |> assign(:champion, load_champion_info(id))
      |> load_win_rates(id, team_position)
-     |> load_summoner_spells(id, team_position)
+     |> load_summoner_spells(id, team_position, patch)
      |> load_items(id, team_position, patch)}
   end
 
@@ -40,13 +40,14 @@ defmodule LoLAnalyticsWeb.ChampionLive.Show do
     end)
   end
 
-  defp load_summoner_spells(socket, champion_id, team_position) do
+  defp load_summoner_spells(socket, champion_id, team_position, patch_number) do
     socket
     |> assign(:summoner_spells, %{status: :loading})
     |> start_async(:get_summoners, fn ->
       LolAnalytics.Facts.ChampionPickedSummonerSpell.Repo.get_champion_picked_summoners(
         champion_id,
-        team_position
+        team_position,
+        patch_number
       )
       |> ShowMapper.map_spells()
     end)
@@ -114,9 +115,6 @@ defmodule LoLAnalyticsWeb.ChampionLive.Show do
   defp page_title(:show), do: "Show Champion"
   defp page_title(:edit), do: "Edit Champion"
 
-  def render_summoner_spells(assigns) do
-  end
-
   def render_items(assigns) do
     case assigns.items do
       %{status: :loading} ->
@@ -151,6 +149,8 @@ defmodule LoLAnalyticsWeb.ChampionLive.Show do
     case assigns.summoner_pells do
       %{status: :loading} ->
         ~H"""
+        <h2 class="text-2xl">Summoner spells</h2>
+        <div class="my-2" />
         <.loader />
         """
 

apps/lol_analytics_web/lib/lol_analytics_web/live/champion_live/show.html.heex

@@ -20,8 +20,11 @@
 <div class="flex flex-row gap-4">
   <.champion_avatar class="w-20" id={@champion.id} name={@champion.name}
     image={"https://ddragon.leagueoflegends.com/cdn/14.11.1/img/champion/#{@champion.image}"} />
-  <div class="win-rate-container">
+  <div>
-    <canvas class="w-full win-rate" height="250" class="win-rate" id="win-rate" phx-hook="ChampionWinRate" />
+    <p class="text-lg">Win rate by patch</p>
+    <div class="win-rate-container" style="position: relative; height:250px; width:400px">
+      <canvas class="w-full win-rate" height="250" class="win-rate" id="win-rate" phx-hook="ChampionWinRate" />
+    </div>
   </div>
 </div>
 <div class="my-4" />

apps/scrapper/lib/scrapper/application.ex

@@ -10,8 +10,8 @@ defmodule Scrapper.Application do
     children = [
       Scrapper.Queue.MatchQueue,
       Scrapper.Queue.PlayerQueue,
-      {Scrapper.Processor.MatchProcessor, []},
+      {Scrapper.Consumer.MatchConsumer, []},
-      {Scrapper.Processor.PlayerProcessor, []}
+      {Scrapper.Consumer.PlayerConsumer, []}
       # Starts a worker by calling: Scrapper.Worker.start_link(arg)
       # {Scrapper.Worker, arg}
     ]

apps/scrapper/lib/scrapper/consumer/match_consumer.ex

@@ -0,0 +1,83 @@
+defmodule Scrapper.Consumer.MatchConsumer do
+  require Logger
+  use Broadway
+
+  def start_link(_opts) do
+    Broadway.start_link(
+      __MODULE__,
+      name: __MODULE__,
+      producer: [
+        module:
+          {BroadwayRabbitMQ.Producer,
+           queue: "match",
+           connection: [
+             username: "guest",
+             password: "guest",
+             host: "localhost"
+           ],
+           on_failure: :reject_and_requeue,
+           qos: [
+             prefetch_count: 1
+           ]},
+        concurrency: 1,
+        rate_limiting: [
+          interval: 300,
+          allowed_messages: 1
+        ]
+      ],
+      processors: [
+        default: [
+          concurrency: 1
+        ]
+      ]
+    )
+  end
+
+  @impl true
+  def handle_message(_, message = %Broadway.Message{}, _) do
+    match_id = message.data
+
+    resp = LoLAPI.MatchApi.get_match_by_id(match_id)
+    process_resp(resp, match_id)
+
+    message
+  end
+
+  def process_resp({:ok, raw_match}, match_id) do
+    Task.start_link(fn ->
+      decoded_match = Poison.decode!(raw_match, as: %LoLAPI.Model.MatchResponse{})
+
+      match_url =
+        case decoded_match.info.queueId do
+          420 ->
+            Logger.info("#{match_id} #{decoded_match.info.gameVersion}")
+
+            Storage.MatchStorage.S3MatchStorage.store_match(
+              match_id,
+              raw_match,
+              "ranked",
+              "#{decoded_match.info.gameVersion}"
+            )
+
+            LolAnalytics.Dimensions.Match.MatchRepo.get_or_create(%{
+              match_id: decoded_match.metadata.matchId,
+              patch_number: decoded_match.info.gameVersion,
+              queue_id: 420
+            })
+
+          _queue_id ->
+            Storage.MatchStorage.S3MatchStorage.store_match(match_id, raw_match, "matches")
+        end
+
+      decoded_match.metadata.participants
+      # |> Enum.shuffle()
+      # |> Enum.take(2)
+      |> Enum.each(fn participant_puuid ->
+        Scrapper.Queue.PlayerQueue.enqueue_puuid(participant_puuid)
+      end)
+    end)
+  end
+
+  def process_resp({:err, _code}, _match_id) do
+  end
+end

apps/scrapper/lib/scrapper/consumer/player_consumer.ex

@@ -1,4 +1,4 @@
-defmodule Scrapper.Processor.PlayerProcessor do
+defmodule Scrapper.Consumer.PlayerConsumer do
   use Broadway
 
   def start_link(_opts) do
@@ -20,7 +20,7 @@ defmodule Scrapper.Processor.PlayerProcessor do
            ]},
         concurrency: 1,
         rate_limiting: [
-          interval: 1000 * 10,
+          interval: 6700,
           allowed_messages: 1
         ]
       ],
@@ -52,7 +52,7 @@ defmodule Scrapper.Processor.PlayerProcessor do
         {
           matches
           |> Enum.each(fn match_id ->
-            Scrapper.Queue.MatchQueue.queue_match(match_id)
+            Scrapper.Queue.MatchQueue.enqueue_match(match_id)
           end)
         }
 

apps/scrapper/lib/scrapper/processor/match_processor.ex

@@ -1,88 +0,0 @@
-defmodule Scrapper.Processor.MatchProcessor do
-  require Logger
-  use Broadway
-
-  def start_link(_opts) do
-    Broadway.start_link(
-      __MODULE__,
-      name: __MODULE__,
-      producer: [
-        module:
-          {BroadwayRabbitMQ.Producer,
-           queue: "match",
-           connection: [
-             username: "guest",
-             password: "guest",
-             host: "localhost"
-           ],
-           on_failure: :reject_and_requeue,
-           qos: [
-             prefetch_count: 1
-           ]},
-        concurrency: 1,
-        rate_limiting: [
-          interval: 333 * 1,
-          allowed_messages: 1
-        ]
-      ],
-      processors: [
-        default: [
-          concurrency: 1
-        ]
-      ]
-    )
-  end
-
-  @impl true
-  def handle_message(_, message = %Broadway.Message{}, _) do
-    match_id = message.data
-
-    resp = LoLAPI.MatchApi.get_match_by_id(match_id)
-    process_resp(resp, match_id)
-
-    message
-  end
-
-  def process_resp({:ok, raw_match}, match_id) do
-    decoded_match = Poison.decode!(raw_match, as: %LoLAPI.Model.MatchResponse{})
-
-    match_url =
-      case decoded_match.info.queueId do
-        420 ->
-          Logger.info("#{match_id} #{decoded_match.info.gameVersion}")
-
-          Storage.MatchStorage.S3MatchStorage.store_match(
-            match_id,
-            raw_match,
-            "ranked",
-            "#{decoded_match.info.gameVersion}"
-          )
-
-        _queue_id ->
-          Storage.MatchStorage.S3MatchStorage.store_match(match_id, raw_match, "matches")
-      end
-
-    match = LolAnalytics.Match.MatchRepo.get_match(match_id)
-
-    case match do
-      nil ->
-        LolAnalytics.Match.MatchRepo.insert_match(match_id)
-
-      _ ->
-        LolAnalytics.Match.MatchRepo.update_match(match, %{
-          :processed => true,
-          :match_url => match_url
-        })
-    end
-
-    decoded_match.metadata.participants
-    |> Enum.shuffle()
-    |> Enum.take(2)
-    |> Enum.each(fn participant_puuid ->
-      Scrapper.Queue.PlayerQueue.queue_puuid(participant_puuid)
-    end)
-  end
-
-  def process_resp({:err, _code}, _match_id) do
-  end
-end

apps/scrapper/lib/scrapper/queue/match_queue.ex

@@ -14,16 +14,16 @@ defmodule Scrapper.Queue.MatchQueue do
     {:ok, %{:channel => channel, :connection => connection}}
   end
 
-  @spec queue_match(String.t()) :: any()
+  @spec enqueue_match(String.t()) :: any()
-  def queue_match(match_id) do
+  def enqueue_match(match_id) do
-    LolAnalytics.Match.MatchRepo.get_match(match_id)
+    LolAnalytics.Dimensions.Match.MatchRepo.get(match_id)
     |> case do
-      nil -> GenServer.call(__MODULE__, {:queue_match, match_id})
+      nil -> GenServer.call(__MODULE__, {:enqueue_match, match_id})
       _match -> :already_processed
     end
   end
 
-  def handle_call({:queue_match, match_id}, _from, %{:channel => channel} = state) do
+  def handle_call({:enqueue_match, match_id}, _from, %{:channel => channel} = state) do
     AMQP.Basic.publish(channel, "", "match", match_id)
     {:reply, nil, state}
   end

apps/scrapper/lib/scrapper/queue/player_queue.ex

@@ -12,31 +12,36 @@ defmodule Scrapper.Queue.PlayerQueue do
   def init(_opts) do
     {:ok, connection} = AMQP.Connection.open()
     {:ok, channel} = AMQP.Channel.open(connection)
-    AMQP.Queue.declare(channel, "player", durable: true)
+
+    AMQP.Queue.declare(channel, "player",
+      durable: true,
+      arguments: [{"x-max-length", :long, 1000}]
+    )
+
     {:ok, {channel, connection}}
   end
 
-  @spec queue_puuid(String.t()) :: nil
+  @spec enqueue_puuid(String.t()) :: nil
-  def queue_puuid(puuid) do
+  def enqueue_puuid(puuid) do
     case LolAnalytics.Player.PlayerRepo.get_player(puuid) do
-      nil -> GenServer.call(__MODULE__, {:queue_player, puuid})
+      nil -> GenServer.call(__MODULE__, {:enqueue_player, puuid})
-      player_entry -> queue_if_not_updated?(player_entry)
+      player_entry -> enqueue_if_not_updated?(player_entry)
     end
   end
 
-  @spec queue_if_not_updated?(%LolAnalytics.Player.PlayerSchema{}) :: nil
+  @spec enqueue_if_not_updated?(%LolAnalytics.Player.PlayerSchema{}) :: nil
-  defp queue_if_not_updated?(player = %LolAnalytics.Player.PlayerSchema{}) do
+  defp enqueue_if_not_updated?(player = %LolAnalytics.Player.PlayerSchema{}) do
     now = DateTime.utc_now() |> DateTime.truncate(:second)
 
     case player.last_processed_at do
       nil ->
-        GenServer.call(__MODULE__, {:queue_player, player.puuid})
+        GenServer.call(__MODULE__, {:enqueue_player, player.puuid})
 
       processed_at ->
         diff_in_hours = DateTime.diff(now, processed_at, :hour)
 
         if diff_in_hours > @hours_to_update_player do
-          GenServer.call(__MODULE__, {:queue_player, player.puuid})
+          GenServer.call(__MODULE__, {:enqueue_player, player.puuid})
         else
           Logger.info(
             "Player #{player.puuid} already processed at #{player.last_processed_at}, diff was #{diff_in_hours}"
@@ -45,7 +50,7 @@ defmodule Scrapper.Queue.PlayerQueue do
     end
   end
 
-  def handle_call({:queue_player, puuid}, _from, {channel, _} = state) do
+  def handle_call({:enqueue_player, puuid}, _from, {channel, _} = state) do
     LolAnalytics.Player.PlayerRepo.insert_player(puuid)
     AMQP.Basic.publish(channel, "", "player", puuid)
     {:reply, nil, state}

config/prod.exs

@@ -8,7 +8,7 @@ import Config
 config :lol_analytics_web, LoLAnalyticsWeb.Endpoint,
   server: true,
   http: [ip: {0, 0, 0, 0}, port: 4000],
-  url: [host: "localhost", port: 80],
+  url: [host: "lol-analytics.kaizer.cloud", port: 443],
   cache_static_manifest: "priv/static/cache_manifest.json"
 
 # Do not print debug messages in production

docker-compose.yml

@@ -26,11 +26,27 @@ services:
       - ~/.docker-conf/rabbitmq/data/:/var/lib/rabbitmq/
       - ~/.docker-conf/rabbitmq/log/:/var/log/rabbitmq
     networks:
-      - rabbitmq_go_net
+      - rabbitmq_net
+  minio:
+    image: bitnami/minio:latest
+    ports:
+      - '9000:9000'
+      - '9001:9001'
+    networks:
+      - minionetwork
+    volumes:
+      - 'minio_data:/data'
+    environment:
+      - MINIO_ROOT_USER=your_username
+      - MINIO_ROOT_PASSWORD=your_pasword
+      - MINIO_DEFAULT_BUCKETS=your_bucket_name
 volumes:
   postgres-data:
+  minio_data:
 
 
 networks:
-  rabbitmq_go_net:
+  rabbitmq_net:
+    driver: bridge
+  minionetwork:
     driver: bridge

mix.lock

@@ -2,7 +2,7 @@
   "amqp": {:hex, :amqp, "3.3.0", "056d9f4bac96c3ab5a904b321e70e78b91ba594766a1fc2f32afd9c016d9f43b", [:mix], [{:amqp_client, "~> 3.9", [hex: :amqp_client, repo: "hexpm", optional: false]}], "hexpm", "8d3ae139d2646c630d674a1b8d68c7f85134f9e8b2a1c3dd5621616994b10a8b"},
   "amqp_client": {:hex, :amqp_client, "3.12.13", "6fc6a7c681e53fed4cbd3f5bcdda342a2b46976345e460ef85414c63698cfe70", [:make, :rebar3], [{:credentials_obfuscation, "3.4.0", [hex: :credentials_obfuscation, repo: "hexpm", optional: false]}, {:rabbit_common, "3.12.13", [hex: :rabbit_common, repo: "hexpm", optional: false]}], "hexpm", "76f41bff0792193f00e0062128db51eb68bcee0eb8236139247a7d1866438d03"},
   "bandit": {:hex, :bandit, "1.5.0", "3bc864a0da7f013ad3713a7f550c6a6ec0e19b8d8715ec678256a0dc197d5539", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "92d18d9a7228a597e0d4661ef69a874ea82d63ff49c7d801a5c68cb18ebbbd72"},
-  "broadway": {:hex, :broadway, "1.0.7", "7808f9e3eb6f53ca6d060f0f9d61012dd8feb0d7a82e62d087dd517b9b66fa53", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e76cfb0a7d64176c387b8b1ddbfb023e2ee8a63e92f43664d78e6d5d0b1177c6"},
+  "broadway": {:hex, :broadway, "1.1.0", "8ed3aea01fd6f5640b3e1515b90eca51c4fc1fac15fb954cdcf75dc054ae719c", [:mix], [{:gen_stage, "~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.7 or ~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "25e315ef1afe823129485d981dcc6d9b221cea30e625fd5439e9b05f44fb60e4"},
   "broadway_rabbitmq": {:hex, :broadway_rabbitmq, "0.8.1", "6d68a480b2e49694e4f3836dcbbf8e621bb97b34e84787a2093d5cc3078a4d87", [:mix], [{:amqp, "~> 1.3 or ~> 2.0 or ~> 3.0", [hex: :amqp, repo: "hexpm", optional: false]}, {:broadway, "~> 1.0", [hex: :broadway, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.3.5 or ~> 0.4.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6eebe9b03c9673cbda790430389c47e4ca867f9904418cff1b71a74a59c2a986"},
   "castore": {:hex, :castore, "1.0.7", "b651241514e5f6956028147fe6637f7ac13802537e895a724f90bf3e36ddd1dd", [:mix], [], "hexpm", "da7785a4b0d2a021cd1292a60875a784b6caef71e76bf4917bdee1f390455cf5"},
   "certifi": {:hex, :certifi, "2.12.0", "2d1cca2ec95f59643862af91f001478c9863c2ac9cb6e2f89780bfd8de987329", [:rebar3], [], "hexpm", "ee68d85df22e554040cdb4be100f33873ac6051387baf6a8f6ce82272340ff1c"},
@@ -21,7 +21,7 @@
   "gen_stage": {:hex, :gen_stage, "1.2.1", "19d8b5e9a5996d813b8245338a28246307fd8b9c99d1237de199d21efc4c76a1", [:mix], [], "hexpm", "83e8be657fa05b992ffa6ac1e3af6d57aa50aace8f691fcf696ff02f8335b001"},
   "gettext": {:hex, :gettext, "0.24.0", "6f4d90ac5f3111673cbefc4ebee96fe5f37a114861ab8c7b7d5b30a1108ce6d8", [:mix], [{:expo, "~> 0.5.1", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "bdf75cdfcbe9e4622dd18e034b227d77dd17f0f133853a1c73b97b3d6c770e8b"},
   "hackney": {:hex, :hackney, "1.20.1", "8d97aec62ddddd757d128bfd1df6c5861093419f8f7a4223823537bad5d064e2", [:rebar3], [{:certifi, "~> 2.12.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "fe9094e5f1a2a2c0a7d10918fee36bfec0ec2a979994cff8cfe8058cd9af38e3"},
-  "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "88ab3a0d790e6a47404cba02800a6b25d2afae50", [tag: "v2.1.1", sparse: "optimized"]},
+  "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "88ab3a0d790e6a47404cba02800a6b25d2afae50", [tag: "v2.1.1", sparse: "optimized", depth: 1]},
   "hpax": {:hex, :hpax, "0.1.2", "09a75600d9d8bbd064cdd741f21fc06fc1f4cf3d0fcc335e5aa19be1a7235c84", [:mix], [], "hexpm", "2c87843d5a23f5f16748ebe77969880e29809580efdaccd615cd3bed628a8c13"},
   "httpoison": {:hex, :httpoison, "2.2.1", "87b7ed6d95db0389f7df02779644171d7319d319178f6680438167d7b69b1f3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "51364e6d2f429d80e14fe4b5f8e39719cacd03eb3f9a9286e61e216feac2d2df"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
@@ -29,7 +29,7 @@
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "2.0.5", "dc34c8efd439abe6ae0343edbb8556f4d63f178594894720607772a041b04b02", [:mix], [], "hexpm", "da0d64a365c45bc9935cc5c8a7fc5e49a0e0f9932a761c55d6c52b142780a05c"},
   "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"},
-  "nimble_options": {:hex, :nimble_options, "1.1.0", "3b31a57ede9cb1502071fade751ab0c7b8dbe75a9a4c2b5bbb0943a690b63172", [:mix], [], "hexpm", "8bbbb3941af3ca9acc7835f5655ea062111c9c27bcac53e004460dfd19008a99"},
+  "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
   "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"},
   "phoenix": {:hex, :phoenix, "1.7.12", "1cc589e0eab99f593a8aa38ec45f15d25297dd6187ee801c8de8947090b5a9d3", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 2.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}, {:phoenix_view, "~> 2.0", [hex: :phoenix_view, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.7", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:plug_crypto, "~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:websock_adapter, "~> 0.5.3", [hex: :websock_adapter, repo: "hexpm", optional: false]}], "hexpm", "d646192fbade9f485b01bc9920c139bfdd19d0f8df3d73fd8eaf2dfbe0d2837c"},
   "phoenix_ecto": {:hex, :phoenix_ecto, "4.5.1", "6fdbc334ea53620e71655664df6f33f670747b3a7a6c4041cdda3e2c32df6257", [:mix], [{:ecto, "~> 3.5", [hex: :ecto, repo: "hexpm", optional: false]}, {:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.1", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:plug, "~> 1.9", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ebe43aa580db129e54408e719fb9659b7f9e0d52b965c5be26cdca416ecead28"},

queries.md

@@ -1,22 +0,0 @@
-```
-SELECT 
-	(cast(count(CASE WHEN is_win THEN 1 END) as float) / cast(count(*) as float)) * 100.0 as win_rate, 
-	count(CASE WHEN is_win THEN 1 END) as games_won, 
-	count(*) as total_games, 
-	champion_id
-	FROM fact_champion_played_game 
-	GROUP BY champion_id
-	ORDER BY win_rate desc;
-```
-
-```
-SELECT 
-	(cast(count(CASE WHEN is_win THEN 1 END) as float) / cast(count(*) as float)) * 100.0 as win_rate, 
-	count(CASE WHEN is_win THEN 1 END) as games_won, 
-	count(*) as total_games, 
-	champion_id,
-    team_position
-	FROM fact_champion_played_game 
-	GROUP BY champion_id, team_position
-	ORDER BY win_rate desc;
-```