wip analytics tables

.kamal/hooks/docker-setup.sample

@@ -1,7 +0,0 @@
-#!/usr/bin/env ruby
-
-# A sample docker-setup hook
-#
-# Sets up a Docker network which can then be used by the application’s containers
-
-ssh user@example.com docker network create kamal

.kamal/hooks/post-deploy.sample

@@ -1,14 +0,0 @@
-#!/bin/sh
-
-# A sample post-deploy hook
-#
-# These environment variables are available:
-# KAMAL_RECORDED_AT
-# KAMAL_PERFORMER
-# KAMAL_VERSION
-# KAMAL_HOSTS
-# KAMAL_ROLE (if set)
-# KAMAL_DESTINATION (if set)
-# KAMAL_RUNTIME
-
-echo "$KAMAL_PERFORMER deployed $KAMAL_VERSION to $KAMAL_DESTINATION in $KAMAL_RUNTIME seconds"

.kamal/hooks/post-traefik-reboot.sample

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-echo "Rebooted Traefik on $KAMAL_HOSTS"

.kamal/hooks/pre-build.sample

@@ -1,51 +0,0 @@
-#!/bin/sh
-
-# A sample pre-build hook
-#
-# Checks:
-# 1. We have a clean checkout
-# 2. A remote is configured
-# 3. The branch has been pushed to the remote
-# 4. The version we are deploying matches the remote
-#
-# These environment variables are available:
-# KAMAL_RECORDED_AT
-# KAMAL_PERFORMER
-# KAMAL_VERSION
-# KAMAL_HOSTS
-# KAMAL_ROLE (if set)
-# KAMAL_DESTINATION (if set)
-
-if [ -n "$(git status --porcelain)" ]; then
-  echo "Git checkout is not clean, aborting..." >&2
-  git status --porcelain >&2
-  exit 1
-fi
-
-first_remote=$(git remote)
-
-if [ -z "$first_remote" ]; then
-  echo "No git remote set, aborting..." >&2
-  exit 1
-fi
-
-current_branch=$(git branch --show-current)
-
-if [ -z "$current_branch" ]; then
-  echo "Not on a git branch, aborting..." >&2
-  exit 1
-fi
-
-remote_head=$(git ls-remote $first_remote --tags $current_branch | cut -f1)
-
-if [ -z "$remote_head" ]; then
-  echo "Branch not pushed to remote, aborting..." >&2
-  exit 1
-fi
-
-if [ "$KAMAL_VERSION" != "$remote_head" ]; then
-  echo "Version ($KAMAL_VERSION) does not match remote HEAD ($remote_head), aborting..." >&2
-  exit 1
-fi
-
-exit 0

.kamal/hooks/pre-connect.sample

@@ -1,47 +0,0 @@
-#!/usr/bin/env ruby
-
-# A sample pre-connect check
-#
-# Warms DNS before connecting to hosts in parallel
-#
-# These environment variables are available:
-# KAMAL_RECORDED_AT
-# KAMAL_PERFORMER
-# KAMAL_VERSION
-# KAMAL_HOSTS
-# KAMAL_ROLE (if set)
-# KAMAL_DESTINATION (if set)
-# KAMAL_RUNTIME
-
-hosts = ENV["KAMAL_HOSTS"].split(",")
-results = nil
-max = 3
-
-elapsed = Benchmark.realtime do
-  results = hosts.map do |host|
-    Thread.new do
-      tries = 1
-
-      begin
-        Socket.getaddrinfo(host, 0, Socket::AF_UNSPEC, Socket::SOCK_STREAM, nil, Socket::AI_CANONNAME)
-      rescue SocketError
-        if tries < max
-          puts "Retrying DNS warmup: #{host}"
-          tries += 1
-          sleep rand
-          retry
-        else
-          puts "DNS warmup failed: #{host}"
-          host
-        end
-      end
-
-      tries
-    end
-  end.map(&:value)
-end
-
-retries = results.sum - hosts.size
-nopes = results.count { |r| r == max }
-
-puts "Prewarmed %d DNS lookups in %.2f sec: %d retries, %d failures" % [ hosts.size, elapsed, retries, nopes ]

.kamal/hooks/pre-deploy.sample

@@ -1,109 +0,0 @@
-#!/usr/bin/env ruby
-
-# A sample pre-deploy hook
-#
-# Checks the Github status of the build, waiting for a pending build to complete for up to 720 seconds.
-#
-# Fails unless the combined status is "success"
-#
-# These environment variables are available:
-# KAMAL_RECORDED_AT
-# KAMAL_PERFORMER
-# KAMAL_VERSION
-# KAMAL_HOSTS
-# KAMAL_COMMAND
-# KAMAL_SUBCOMMAND
-# KAMAL_ROLE (if set)
-# KAMAL_DESTINATION (if set)
-
-# Only check the build status for production deployments
-if ENV["KAMAL_COMMAND"] == "rollback" || ENV["KAMAL_DESTINATION"] != "production"
-  exit 0
-end
-
-require "bundler/inline"
-
-# true = install gems so this is fast on repeat invocations
-gemfile(true, quiet: true) do
-  source "https://rubygems.org"
-
-  gem "octokit"
-  gem "faraday-retry"
-end
-
-MAX_ATTEMPTS = 72
-ATTEMPTS_GAP = 10
-
-def exit_with_error(message)
-  $stderr.puts message
-  exit 1
-end
-
-class GithubStatusChecks
-  attr_reader :remote_url, :git_sha, :github_client, :combined_status
-
-  def initialize
-    @remote_url = `git config --get remote.origin.url`.strip.delete_prefix("https://github.com/")
-    @git_sha = `git rev-parse HEAD`.strip
-    @github_client = Octokit::Client.new(access_token: ENV["GITHUB_TOKEN"])
-    refresh!
-  end
-
-  def refresh!
-    @combined_status = github_client.combined_status(remote_url, git_sha)
-  end
-
-  def state
-    combined_status[:state]
-  end
-
-  def first_status_url
-    first_status = combined_status[:statuses].find { |status| status[:state] == state }
-    first_status && first_status[:target_url]
-  end
-
-  def complete_count
-    combined_status[:statuses].count { |status| status[:state] != "pending"}
-  end
-
-  def total_count
-    combined_status[:statuses].count
-  end
-
-  def current_status
-    if total_count > 0
-      "Completed #{complete_count}/#{total_count} checks, see #{first_status_url} ..."
-    else
-      "Build not started..."
-    end
-  end
-end
-
-
-$stdout.sync = true
-
-puts "Checking build status..."
-attempts = 0
-checks = GithubStatusChecks.new
-
-begin
-  loop do
-    case checks.state
-    when "success"
-      puts "Checks passed, see #{checks.first_status_url}"
-      exit 0
-    when "failure"
-      exit_with_error "Checks failed, see #{checks.first_status_url}"
-    when "pending"
-      attempts += 1
-    end
-
-    exit_with_error "Checks are still pending, gave up after #{MAX_ATTEMPTS * ATTEMPTS_GAP} seconds" if attempts == MAX_ATTEMPTS
-
-    puts checks.current_status
-    sleep(ATTEMPTS_GAP)
-    checks.refresh!
-  end
-rescue Octokit::NotFound
-  exit_with_error "Build status could not be found"
-end

.kamal/hooks/pre-traefik-reboot.sample

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-echo "Rebooting Traefik on $KAMAL_HOSTS..."

apps/lol_analytics/lib/lol_analytics/analyzer/champion_analyzer.ex

@@ -13,8 +13,8 @@ defmodule LolAnalytics.Analyzer.ChampionAnalyzer do
   @doc """
   iex> LolAnalytics.Analyzer.ChampionAnalyzer.analyze(:url, "http://localhost:9000/ranked/14.9.580.2108/EUW1_6923309745.json")
   """
+  @spec analyze(any(), any()) :: none()
   @impl true
-  @spec analyze(atom(), String.t()) :: :ok
   def analyze(:url, path) do
     data = HTTPoison.get!(path)
     analyze(:data, data.body)
@@ -22,7 +22,6 @@ defmodule LolAnalytics.Analyzer.ChampionAnalyzer do
   end
 
   @impl true
-  @spec analyze(atom(), any()) :: list(LoLAPI.Model.Participant.t())
   def analyze(:data, data) do
     decoded_match = Poison.decode!(data, as: %LoLAPI.Model.MatchResponse{})
     participants = decoded_match.info.participants

apps/lol_analytics/lib/lol_analytics/facts/champion_played_game/champion_played_game_repo.ex

@@ -0,0 +1,3 @@
+defmodule LolAnalytics.Facts.ChampionPlayedGame.ChampionPlayedGameRepo do
+
+end

apps/lol_analytics/lib/lol_analytics/facts/champion_played_game/champion_played_game_schema.ex

@@ -0,0 +1,11 @@
+defmodule LolAnalytics.Facts.ChampionPlayedGame.ChampionPlayedGameSchema do
+  use Ecto.Schema
+
+  schema "fact_champion_played_game" do
+    field :champion_id, :integer
+    field :match_id, :integer
+    field :is_win, :boolean
+    field :game_length_seconds, :integer
+    field :queue_id, :integer
+  end
+end

apps/lol_analytics/priv/repo/migrations/20240526160430_analytics_tables.exs

@@ -0,0 +1,51 @@
+defmodule LoLAnalytics.Repo.Migrations.AnalyticsTables do
+  use Ecto.Migration
+
+  def change do
+    create table("dom_champion") do
+      add :champion_id, :integer, primary_key: true
+      timestamps()
+    end
+
+    create table("dom_match") do
+      add :match_id, :integer, primary_key: true
+      timestamps()
+    end
+
+    create index("dom_match", [:match_id], unique: true)
+
+    create table("dom_patch") do
+      add :patch_number, :string, primary_key: true
+      timestamps()
+    end
+
+    create index("dom_patch", [:patch_number], unique: true)
+
+    create table("dom_item") do
+      add :item_id, :integer, primary_key: true
+      timestamps()
+    end
+
+    create index("dom_item", [:item_id], unique: true)
+
+    create table("dom_summoner_spell") do
+      add :spell_id, :integer, primary_key: true
+      timestamps()
+    end
+
+    create index("dom_summoner_spell", [:spell_id], unique: true)
+
+    create table("fact_champion_played_game") do
+      add :champion_id, references("dom_champion", with: [champion_id: :champion_id]),
+        primary_key: true
+
+      add :match_id, references("dom_match", with: [match_id: :match_id])
+      add :is_win, :boolean
+      add :game_length_seconds, :integer
+      add :queue_id, :integer
+      timestamps()
+    end
+
+    create index("fact_champion_played_game", [:id, :champion_id, :queue_id])
+  end
+end

apps/scrapper/lib/scrapper/match_classifier.ex

@@ -6,6 +6,29 @@ defmodule Scrapper.MatchClassifier do
     classify_match_by_queue(match.info.queueId)
   end
 
+  def stream_classify_matches_by_queue(queue \\ 420, bucket \\ "ranked") do
+    Storage.MatchStorage.S3MatchStorage.stream_files("matches")
+    |> Stream.each(fn match ->
+      %{key: json_file} = match
+      [key | _] = String.split(json_file, ".")
+
+      response =
+        HTTPoison.get!("http://#{System.get_env("EX_AWS_ENDPOINT")}:9000/matches/#{key}.json", [],
+          timeout: 5000
+        )
+
+      %{"info" => %{"gameVersion" => gameVersion, "queueId" => queueId}} =
+        Poison.decode!(response.body)
+
+      if queueId == queue do
+        Storage.MatchStorage.S3MatchStorage.store_match(key, response.body, bucket, gameVersion)
+        Logger.info("Match #{key} processed")
+      end
+
+      match
+    end)
+  end
+
   @spec classify_match_by_queue(String.t()) :: nil
   def classify_match_by_queue("420") do
     matches = Storage.MatchStorage.S3MatchStorage.list_files("matches")

apps/storage/lib/match_storage/s3_match_storage.ex

@@ -6,6 +6,11 @@ defmodule Storage.MatchStorage.S3MatchStorage do
     ""
   end
 
+  def stream_files(path) do
+    ExAws.S3.list_objects_v2(path)
+    |> ExAws.stream!()
+  end
+
   @doc """
   Lists all files at the given path.
 

config/deploy.yml

@@ -1,101 +0,0 @@
-# Name of your application. Used to uniquely configure containers.
-service: my-app
-
-# Name of the container image.
-image: user/my-app
-
-# Deploy to these servers.
-servers:
-  - 192.168.0.1
-
-# Credentials for your image host.
-registry:
-  # Specify the registry server, if you're not using Docker Hub
-  # server: registry.digitalocean.com / ghcr.io / ...
-  username: my-user
-
-  # Always use an access token rather than real password when possible.
-  password:
-    - KAMAL_REGISTRY_PASSWORD
-
-# Inject ENV variables into containers (secrets come from .env).
-# Remember to run `kamal env push` after making changes!
-# env:
-#   clear:
-#     DB_HOST: 192.168.0.2
-#   secret:
-#     - RAILS_MASTER_KEY
-
-# Use a different ssh user than root
-# ssh:
-#   user: app
-
-# Configure builder setup.
-# builder:
-#   args:
-#     RUBY_VERSION: 3.2.0
-#   secrets:
-#     - GITHUB_TOKEN
-#   remote:
-#     arch: amd64
-#     host: ssh://app@192.168.0.1
-
-# Use accessory services (secrets come from .env).
-# accessories:
-#   db:
-#     image: mysql:8.0
-#     host: 192.168.0.2
-#     port: 3306
-#     env:
-#       clear:
-#         MYSQL_ROOT_HOST: '%'
-#       secret:
-#         - MYSQL_ROOT_PASSWORD
-#     files:
-#       - config/mysql/production.cnf:/etc/mysql/my.cnf
-#       - db/production.sql:/docker-entrypoint-initdb.d/setup.sql
-#     directories:
-#       - data:/var/lib/mysql
-#   redis:
-#     image: redis:7.0
-#     host: 192.168.0.2
-#     port: 6379
-#     directories:
-#       - data:/data
-
-# Configure custom arguments for Traefik. Be sure to reboot traefik when you modify it.
-# traefik:
-#   args:
-#     accesslog: true
-#     accesslog.format: json
-
-# Configure a custom healthcheck (default is /up on port 3000)
-# healthcheck:
-#   path: /healthz
-#   port: 4000
-
-# Bridge fingerprinted assets, like JS and CSS, between versions to avoid
-# hitting 404 on in-flight requests. Combines all files from new and old
-# version inside the asset_path.
-#
-# If your app is using the Sprockets gem, ensure it sets `config.assets.manifest`.
-# See https://github.com/basecamp/kamal/issues/626 for details
-#
-# asset_path: /rails/public/assets
-
-# Configure rolling deploys by setting a wait time between batches of restarts.
-# boot:
-#   limit: 10 # Can also specify as a percentage of total hosts, such as "25%"
-#   wait: 2
-
-# Configure the role used to determine the primary_host. This host takes
-# deploy locks, runs health checks during the deploy, and follow logs, etc.
-#
-# Caution: there's no support for role renaming yet, so be careful to cleanup
-#          the previous role on the deployed hosts.
-# primary_role: web
-
-# Controls if we abort when see a role with no hosts. Disabling this may be
-# useful for more complex deploy configurations.
-#
-# allow_empty_roles: false