diff --git a/apps/scrapper/lib/scrapper/match_classifier.ex b/apps/scrapper/lib/scrapper/match_classifier.ex index cade2d7..ea9737c 100644 --- a/apps/scrapper/lib/scrapper/match_classifier.ex +++ b/apps/scrapper/lib/scrapper/match_classifier.ex @@ -6,6 +6,29 @@ defmodule Scrapper.MatchClassifier do classify_match_by_queue(match.info.queueId) end + def stream_classify_matches_by_queue(queue \\ 420, bucket \\ "ranked") do + Storage.MatchStorage.S3MatchStorage.stream_files("matches") + |> Stream.each(fn match -> + %{key: json_file} = match + [key | _] = String.split(json_file, ".") + + response = + HTTPoison.get!("http://#{System.get_env("EX_AWS_ENDPOINT")}:9000/matches/#{key}.json", [], + timeout: 5000 + ) + + %{"info" => %{"gameVersion" => gameVersion, "queueId" => queueId}} = + Poison.decode!(response.body) + + if queueId == queue do + Storage.MatchStorage.S3MatchStorage.store_match(key, response.body, bucket, gameVersion) + Logger.info("Match #{key} processed") + end + + match + end) + end + @spec classify_match_by_queue(String.t()) :: nil def classify_match_by_queue("420") do matches = Storage.MatchStorage.S3MatchStorage.list_files("matches") diff --git a/apps/storage/lib/match_storage/s3_match_storage.ex b/apps/storage/lib/match_storage/s3_match_storage.ex index 7f4edfe..391064c 100644 --- a/apps/storage/lib/match_storage/s3_match_storage.ex +++ b/apps/storage/lib/match_storage/s3_match_storage.ex @@ -6,6 +6,11 @@ defmodule Storage.MatchStorage.S3MatchStorage do "" end + def stream_files(path) do + ExAws.S3.list_objects_v2(path) + |> ExAws.stream!() + end + @doc """ Lists all files at the given path.