classify matches with stream

apps/scrapper/lib/scrapper/match_classifier.ex

@@ -6,6 +6,29 @@ defmodule Scrapper.MatchClassifier do
     classify_match_by_queue(match.info.queueId)
   end
 
+  def stream_classify_matches_by_queue(queue \\ 420, bucket \\ "ranked") do
+    Storage.MatchStorage.S3MatchStorage.stream_files("matches")
+    |> Stream.each(fn match ->
+      %{key: json_file} = match
+      [key | _] = String.split(json_file, ".")
+
+      response =
+        HTTPoison.get!("http://#{System.get_env("EX_AWS_ENDPOINT")}:9000/matches/#{key}.json", [],
+          timeout: 5000
+        )
+
+      %{"info" => %{"gameVersion" => gameVersion, "queueId" => queueId}} =
+        Poison.decode!(response.body)
+
+      if queueId == queue do
+        Storage.MatchStorage.S3MatchStorage.store_match(key, response.body, bucket, gameVersion)
+        Logger.info("Match #{key} processed")
+      end
+
+      match
+    end)
+  end
+
   @spec classify_match_by_queue(String.t()) :: nil
   def classify_match_by_queue("420") do
     matches = Storage.MatchStorage.S3MatchStorage.list_files("matches")

apps/storage/lib/match_storage/s3_match_storage.ex

@@ -6,6 +6,11 @@ defmodule Storage.MatchStorage.S3MatchStorage do
     ""
   end
 
+  def stream_files(path) do
+    ExAws.S3.list_objects_v2(path)
+    |> ExAws.stream!()
+  end
+
   @doc """
   Lists all files at the given path.