Store matches to object storage

apps/scrapper/config/libs/ex_aws_dev.exs

@@ -1,8 +1,17 @@
 import Config
 
-config :ex_aws, :s3,
-  scheme: "http://",
-  host: System.get_env("EX_AWS_HOST"),
-  port: System.get_env("EX_AWS_PORT"),
-  access_key_id: System.get_env("EX_AWS_ACCESS_KEY"),
-  secret_access_key: System.get_env("EX_AWS_SECRET_KEY")
+# config :ex_aws, :s3,
+#   scheme: "http://",
+#   host: System.get_env("EX_AWS_HOST"),
+#   port: System.get_env("EX_AWS_PORT"),
+#   access_key_id: System.get_env("EX_AWS_ACCESS_KEY"),
+#   secret_access_key: System.get_env("EX_AWS_SECRET_KEY")
+
+config :ex_aws,
+  access_key_id: "3zwMWl4RPCs8CHzhKmIX",
+  secret_access_key: "79B6LmryjJElkrIiHgDcfIxSpmvrLdVy75MyAJC2",
+  s3: [
+    scheme: "http://",
+    host: "localhost",
+    port: "9000"
+  ]

apps/scrapper/lib/scrapper/api/account_api.ex

@@ -0,0 +1,4 @@
+defmodule Scrapper.Api.AccountApi do
+  def get_puuid(name, tag) do
+  end
+end

apps/scrapper/lib/scrapper/api/match_api.ex

@@ -1,4 +1,5 @@
 defmodule Scrapper.Data.Api.MatchApi do
+  import Logger
   @match_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/%{matchid}"
   @puuid_matches_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/by-puuid/%{puuid}/ids"
 
@@ -7,7 +8,7 @@ defmodule Scrapper.Data.Api.MatchApi do
 
   iex> Scrapper.Data.MatchApi.get_match_by_id("EUW1_6921743825")
   """
-  @spec get_match_by_id(String.t()) :: %Scrapper.Data.Api.Model.Match.MatchResponse{}
+  @spec get_match_by_id(String.t()) :: %Scrapper.Api.Model.MatchResponse{}
   def get_match_by_id(match_id) do
     url = String.replace(@match_base_endpoint, "%{matchid}", match_id)
     api_key = System.get_env("RIOT_API_KEY")
@@ -16,13 +17,11 @@ defmodule Scrapper.Data.Api.MatchApi do
 
     case response.status_code do
       200 ->
-        # process the response here
-        response.body
-        Poison.decode!(response.body, as: %Scrapper.Data.Api.Model.Match.MatchResponse{})
+        {:ok, response.body}
 
       _ ->
-        # handle error responses
-        IO.inspect(response.status_code)
+        Logger.error("Error getting match by id: #{match_id} #{inspect(response)}")
+        {:err, response.status_code}
     end
   end
 
@@ -35,13 +34,11 @@ defmodule Scrapper.Data.Api.MatchApi do
 
     case response.status_code do
       200 ->
-        # process the response here
-        IO.inspect(response.body)
-        Poison.decode!(response.body)
+        {:ok, Poison.decode!(response.body)}
 
-      _ ->
-        # handle error responses
-        IO.inspect(response.status_code)
+      code ->
+        Logger.error("Error getting matches from player #{puuid} #{code}")
+        {:err, response.status_code}
     end
   end
 end

apps/scrapper/lib/scrapper/api/model/info.ex

@@ -1,5 +1,5 @@
-defmodule Scrapper.Data.Api.Model.Match.Info do
-  alias Scrapper.Data.Api.Model.Match.Participant
+defmodule Scrapper.Api.Model.Info do
+  alias Scrapper.Api.Model.Participant
 
   defstruct endOfGameResult: "",
             gameCreation: "",

apps/scrapper/lib/scrapper/api/model/match_response.ex

@@ -0,0 +1,6 @@
+defmodule Scrapper.Api.Model.MatchResponse do
+  alias Scrapper.Api.Model.{Info, Metadata}
+
+  defstruct metadata: %Metadata{},
+            info: %Info{}
+end

apps/scrapper/lib/scrapper/api/model/metadata.ex

@@ -1,3 +1,3 @@
-defmodule Scrapper.Data.Api.Model.Match.Metadata do
+defmodule Scrapper.Api.Model.Metadata do
   defstruct [:dataVersion, :matchId, :participants]
 end

apps/scrapper/lib/scrapper/api/model/participant.ex

@@ -1,4 +1,4 @@
-defmodule Scrapper.Data.Api.Model.Match.Participant do
+defmodule Scrapper.Api.Model.Participant do
   # Enum.map(participant, fn {k,_v} -> ":#{k}" end) |> Enum.join(", ")
   defstruct [
     :onMyWayPings,

apps/scrapper/lib/scrapper/data/api/match/match_response.ex

@@ -1,6 +0,0 @@
-defmodule Scrapper.Data.Api.Model.Match.MatchResponse do
-  alias Scrapper.Data.Api.Model.Match.{Info, Metadata}
-
-  defstruct metadata: %Metadata{},
-            info: %Info{}
-end

apps/scrapper/lib/scrapper/data/storage/s3_match_storage.ex

@@ -1,11 +0,0 @@
-defmodule Scrapper.Data.Storage.S3MatchStorage do
-  @behaviour Scrapper.Data.Storage.MatchStorage
-
-  def get_match(match_id) do
-    ""
-  end
-
-  def store_match(match_id, match_data) do
-    :ok
-  end
-end

apps/scrapper/lib/scrapper/processor/match_processor.ex

@@ -20,8 +20,8 @@ defmodule Scrapper.Processor.MatchProcessor do
            ]},
         concurrency: 1,
         rate_limiting: [
-          interval: 1000 * 90,
-          allowed_messages: 5
+          interval: 1000 * 60,
+          allowed_messages: 150
         ]
       ],
       processors: [
@@ -35,20 +35,32 @@ defmodule Scrapper.Processor.MatchProcessor do
   @impl true
   def handle_message(_, message = %Broadway.Message{}, _) do
     match_id = message.data
-    IO.inspect(match_id)
 
-    match = Scrapper.Data.Api.MatchApi.get_match_by_id(match_id)
+    resp = Scrapper.Data.Api.MatchApi.get_match_by_id(match_id)
+    process_resp(resp, match_id)
 
-    match.metadata.participants
-    |> Enum.each(fn participant ->
-      Scrapper.Data.Api.MatchApi.get_matches_from_player(participant)
-      |> Enum.each(fn match_id ->
-        nil
-        Scrapper.Queue.MatchQueue.queue_match(match_id)
-      end)
-    end)
-
-    IO.inspect(match.info.participants)
     message
   end
+
+  def process_resp({:ok, raw_match}, match_id) do
+    decoded_match = Poison.decode!(raw_match, as: %Scrapper.Api.Model.MatchResponse{})
+    Scrapper.Storage.S3MatchStorage.store_match(match_id, raw_match)
+
+    decoded_match.metadata.participants
+    |> Enum.each(fn participant ->
+      case Scrapper.Data.Api.MatchApi.get_matches_from_player(participant) do
+        {:ok, matches} ->
+          matches
+          |> Enum.each(fn match_id ->
+            Scrapper.Queue.MatchQueue.queue_match(match_id)
+          end)
+
+        {:err, code} ->
+          {:err, code}
+      end
+    end)
+  end
+
+  def process_resp({:err, code}, match_id) do
+  end
 end

apps/scrapper/lib/scrapper/processor/player_processor.ex

@@ -20,13 +20,13 @@ defmodule Scrapper.Processor.PlayerProcessor do
            ]},
         concurrency: 1,
         rate_limiting: [
-          interval: 1000 * 90,
-          allowed_messages: 2
+          interval: 1000 * 60,
+          allowed_messages: 5
         ]
       ],
       processors: [
         default: [
-          concurrency: 2
+          concurrency: 5
         ]
       ]
     )
@@ -36,13 +36,20 @@ defmodule Scrapper.Processor.PlayerProcessor do
   def handle_message(_, message = %Broadway.Message{}, _) do
     puuid = message.data
 
-    IO.inspect(puuid)
+    resp = Scrapper.Data.Api.MatchApi.get_matches_from_player(puuid)
 
-    Scrapper.Data.Api.MatchApi.get_matches_from_player(puuid)
-    |> Enum.each(fn match_id ->
-      IO.inspect(match_id)
-      Scrapper.Queue.MatchQueue.queue_match(match_id)
-    end)
+    case resp do
+      {:ok, matches} ->
+        {
+          matches
+          |> Enum.each(fn match_id ->
+            Scrapper.Queue.MatchQueue.queue_match(match_id)
+          end)
+        }
+
+      {:err, code} ->
+        {:err, code}
+    end
 
     message
   end

apps/scrapper/lib/scrapper/queue/match_queue.ex

@@ -18,7 +18,7 @@ defmodule Scrapper.Queue.MatchQueue do
     GenServer.call(__MODULE__, {:queue_match, match_id})
   end
 
-  def handle_call({:queue_match, match_id}, from, %{:channel => channel} = state) do
+  def handle_call({:queue_match, match_id}, _from, %{:channel => channel} = state) do
     AMQP.Basic.publish(channel, "", "match", match_id)
     {:reply, nil, state}
   end

apps/scrapper/lib/scrapper/storage/s3_match_storage.ex

@@ -0,0 +1,33 @@
+defmodule Scrapper.Storage.S3MatchStorage do
+  require Logger
+  import SweetXml
+
+  @behaviour Scrapper.Data.Storage.MatchStorage
+
+  def get_match(match_id) do
+    ""
+  end
+
+  @doc """
+
+  iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content"
+  """
+  @spec store_match(String.t(), String.t()) :: none()
+  def store_match(match_id, match_data) do
+    File.write("/tmp/#{match_id}.json", match_data)
+
+    url =
+      "/tmp/#{match_id}.json"
+      |> ExAws.S3.Upload.stream_file()
+      |> ExAws.S3.upload("matches", "#{match_id}.json")
+      |> ExAws.request!()
+      |> extract_s3_url_from_upload
+
+    Logger.info("Stored match at #{url}")
+
+    url
+  end
+
+  defp extract_s3_url_from_upload(%{:body => %{:location => location}} = _s3_response),
+    do: location
+end

apps/scrapper/mix.exs

@@ -29,11 +29,12 @@ defmodule Scrapper.MixProject do
       {:httpoison, "~> 2.2"},
       {:poison, "~> 5.0"},
       {:ex_aws, "~> 2.1"},
-      {:ex_aws_s3, "~> 2.0"},
+      {:ex_aws_s3, "~> 2.5.3"},
       {:hackney, "~> 1.9"},
       {:sweet_xml, "~> 0.6"},
       {:broadway_rabbitmq, "~> 0.7"},
-      {:amqp, "~> 3.3"}
+      {:amqp, "~> 3.3"},
+      {:sweet_xml, "~> 0.7.3"}
       # {:dep_from_hexpm, "~> 0.3.0"},
       # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"},
       # {:sibling_app_in_umbrella, in_umbrella: true}