wip analytics tables

apps/lol_analytics/lib/lol_analytics/analyzer/champion_analyzer.ex

@@ -13,8 +13,8 @@ defmodule LolAnalytics.Analyzer.ChampionAnalyzer do
   @doc """
   iex> LolAnalytics.Analyzer.ChampionAnalyzer.analyze(:url, "http://localhost:9000/ranked/14.9.580.2108/EUW1_6923309745.json")
   """
+  @spec analyze(any(), any()) :: none()
   @impl true
-  @spec analyze(atom(), String.t()) :: :ok
   def analyze(:url, path) do
     data = HTTPoison.get!(path)
     analyze(:data, data.body)
@@ -22,7 +22,6 @@ defmodule LolAnalytics.Analyzer.ChampionAnalyzer do
   end
 
   @impl true
-  @spec analyze(atom(), any()) :: list(LoLAPI.Model.Participant.t())
   def analyze(:data, data) do
     decoded_match = Poison.decode!(data, as: %LoLAPI.Model.MatchResponse{})
     participants = decoded_match.info.participants

apps/lol_analytics/lib/lol_analytics/facts/champion_played_game/champion_played_game_repo.ex

@@ -0,0 +1,3 @@
+defmodule LolAnalytics.Facts.ChampionPlayedGame.ChampionPlayedGameRepo do
+
+end

apps/lol_analytics/lib/lol_analytics/facts/champion_played_game/champion_played_game_schema.ex

@@ -0,0 +1,11 @@
+defmodule LolAnalytics.Facts.ChampionPlayedGame.ChampionPlayedGameSchema do
+  use Ecto.Schema
+
+  schema "fact_champion_played_game" do
+    field :champion_id, :integer
+    field :match_id, :integer
+    field :is_win, :boolean
+    field :game_length_seconds, :integer
+    field :queue_id, :integer
+  end
+end

apps/lol_analytics/priv/repo/migrations/20240526160430_analytics_tables.exs

@@ -0,0 +1,51 @@
+defmodule LoLAnalytics.Repo.Migrations.AnalyticsTables do
+  use Ecto.Migration
+
+  def change do
+    create table("dom_champion") do
+      add :champion_id, :integer, primary_key: true
+      timestamps()
+    end
+
+    create table("dom_match") do
+      add :match_id, :integer, primary_key: true
+      timestamps()
+    end
+
+    create index("dom_match", [:match_id], unique: true)
+
+    create table("dom_patch") do
+      add :patch_number, :string, primary_key: true
+      timestamps()
+    end
+
+    create index("dom_patch", [:patch_number], unique: true)
+
+    create table("dom_item") do
+      add :item_id, :integer, primary_key: true
+      timestamps()
+    end
+
+    create index("dom_item", [:item_id], unique: true)
+
+    create table("dom_summoner_spell") do
+      add :spell_id, :integer, primary_key: true
+      timestamps()
+    end
+
+    create index("dom_summoner_spell", [:spell_id], unique: true)
+
+    create table("fact_champion_played_game") do
+      add :champion_id, references("dom_champion", with: [champion_id: :champion_id]),
+        primary_key: true
+
+      add :match_id, references("dom_match", with: [match_id: :match_id])
+      add :is_win, :boolean
+      add :game_length_seconds, :integer
+      add :queue_id, :integer
+      timestamps()
+    end
+
+    create index("fact_champion_played_game", [:id, :champion_id, :queue_id])
+  end
+end

apps/scrapper/lib/scrapper/match_classifier.ex

@@ -6,6 +6,29 @@ defmodule Scrapper.MatchClassifier do
     classify_match_by_queue(match.info.queueId)
   end
 
+  def stream_classify_matches_by_queue(queue \\ 420, bucket \\ "ranked") do
+    Storage.MatchStorage.S3MatchStorage.stream_files("matches")
+    |> Stream.each(fn match ->
+      %{key: json_file} = match
+      [key | _] = String.split(json_file, ".")
+
+      response =
+        HTTPoison.get!("http://#{System.get_env("EX_AWS_ENDPOINT")}:9000/matches/#{key}.json", [],
+          timeout: 5000
+        )
+
+      %{"info" => %{"gameVersion" => gameVersion, "queueId" => queueId}} =
+        Poison.decode!(response.body)
+
+      if queueId == queue do
+        Storage.MatchStorage.S3MatchStorage.store_match(key, response.body, bucket, gameVersion)
+        Logger.info("Match #{key} processed")
+      end
+
+      match
+    end)
+  end
+
   @spec classify_match_by_queue(String.t()) :: nil
   def classify_match_by_queue("420") do
     matches = Storage.MatchStorage.S3MatchStorage.list_files("matches")

apps/storage/lib/match_storage/s3_match_storage.ex

@@ -6,6 +6,11 @@ defmodule Storage.MatchStorage.S3MatchStorage do
     ""
   end
 
+  def stream_files(path) do
+    ExAws.S3.list_objects_v2(path)
+    |> ExAws.stream!()
+  end
+
   @doc """
   Lists all files at the given path.