diff --git a/apps/scrapper/config/libs/ex_aws_dev.exs b/apps/scrapper/config/libs/ex_aws_dev.exs index 7449999..e0292c1 100644 --- a/apps/scrapper/config/libs/ex_aws_dev.exs +++ b/apps/scrapper/config/libs/ex_aws_dev.exs @@ -1,8 +1,17 @@ import Config -config :ex_aws, :s3, - scheme: "http://", - host: System.get_env("EX_AWS_HOST"), - port: System.get_env("EX_AWS_PORT"), - access_key_id: System.get_env("EX_AWS_ACCESS_KEY"), - secret_access_key: System.get_env("EX_AWS_SECRET_KEY") +# config :ex_aws, :s3, +# scheme: "http://", +# host: System.get_env("EX_AWS_HOST"), +# port: System.get_env("EX_AWS_PORT"), +# access_key_id: System.get_env("EX_AWS_ACCESS_KEY"), +# secret_access_key: System.get_env("EX_AWS_SECRET_KEY") + +config :ex_aws, + access_key_id: "3zwMWl4RPCs8CHzhKmIX", + secret_access_key: "79B6LmryjJElkrIiHgDcfIxSpmvrLdVy75MyAJC2", + s3: [ + scheme: "http://", + host: "localhost", + port: "9000" + ] diff --git a/apps/scrapper/lib/scrapper/api/account_api.ex b/apps/scrapper/lib/scrapper/api/account_api.ex new file mode 100644 index 0000000..b079143 --- /dev/null +++ b/apps/scrapper/lib/scrapper/api/account_api.ex @@ -0,0 +1,4 @@ +defmodule Scrapper.Api.AccountApi do + def get_puuid(name, tag) do + end +end diff --git a/apps/scrapper/lib/scrapper/data/api/match_api.ex b/apps/scrapper/lib/scrapper/api/match_api.ex similarity index 68% rename from apps/scrapper/lib/scrapper/data/api/match_api.ex rename to apps/scrapper/lib/scrapper/api/match_api.ex index 1f6e2ab..aea8cde 100644 --- a/apps/scrapper/lib/scrapper/data/api/match_api.ex +++ b/apps/scrapper/lib/scrapper/api/match_api.ex @@ -1,4 +1,5 @@ defmodule Scrapper.Data.Api.MatchApi do + import Logger @match_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/%{matchid}" @puuid_matches_base_endpoint "https://europe.api.riotgames.com/lol/match/v5/matches/by-puuid/%{puuid}/ids" @@ -7,7 +8,7 @@ defmodule Scrapper.Data.Api.MatchApi do iex> Scrapper.Data.MatchApi.get_match_by_id("EUW1_6921743825") """ - @spec get_match_by_id(String.t()) :: %Scrapper.Data.Api.Model.Match.MatchResponse{} + @spec get_match_by_id(String.t()) :: %Scrapper.Api.Model.MatchResponse{} def get_match_by_id(match_id) do url = String.replace(@match_base_endpoint, "%{matchid}", match_id) api_key = System.get_env("RIOT_API_KEY") @@ -16,13 +17,11 @@ defmodule Scrapper.Data.Api.MatchApi do case response.status_code do 200 -> - # process the response here - response.body - Poison.decode!(response.body, as: %Scrapper.Data.Api.Model.Match.MatchResponse{}) + {:ok, response.body} _ -> - # handle error responses - IO.inspect(response.status_code) + Logger.error("Error getting match by id: #{match_id} #{inspect(response)}") + {:err, response.status_code} end end @@ -35,13 +34,11 @@ defmodule Scrapper.Data.Api.MatchApi do case response.status_code do 200 -> - # process the response here - IO.inspect(response.body) - Poison.decode!(response.body) + {:ok, Poison.decode!(response.body)} - _ -> - # handle error responses - IO.inspect(response.status_code) + code -> + Logger.error("Error getting matches from player #{puuid} #{code}") + {:err, response.status_code} end end end diff --git a/apps/scrapper/lib/scrapper/data/api/match/info.ex b/apps/scrapper/lib/scrapper/api/model/info.ex similarity index 82% rename from apps/scrapper/lib/scrapper/data/api/match/info.ex rename to apps/scrapper/lib/scrapper/api/model/info.ex index 835bccc..6115fe7 100644 --- a/apps/scrapper/lib/scrapper/data/api/match/info.ex +++ b/apps/scrapper/lib/scrapper/api/model/info.ex @@ -1,5 +1,5 @@ -defmodule Scrapper.Data.Api.Model.Match.Info do - alias Scrapper.Data.Api.Model.Match.Participant +defmodule Scrapper.Api.Model.Info do + alias Scrapper.Api.Model.Participant defstruct endOfGameResult: "", gameCreation: "", diff --git a/apps/scrapper/lib/scrapper/api/model/match_response.ex b/apps/scrapper/lib/scrapper/api/model/match_response.ex new file mode 100644 index 0000000..da92b9d --- /dev/null +++ b/apps/scrapper/lib/scrapper/api/model/match_response.ex @@ -0,0 +1,6 @@ +defmodule Scrapper.Api.Model.MatchResponse do + alias Scrapper.Api.Model.{Info, Metadata} + + defstruct metadata: %Metadata{}, + info: %Info{} +end diff --git a/apps/scrapper/lib/scrapper/data/api/match/metadata.ex b/apps/scrapper/lib/scrapper/api/model/metadata.ex similarity index 51% rename from apps/scrapper/lib/scrapper/data/api/match/metadata.ex rename to apps/scrapper/lib/scrapper/api/model/metadata.ex index ad34572..1dbfe00 100644 --- a/apps/scrapper/lib/scrapper/data/api/match/metadata.ex +++ b/apps/scrapper/lib/scrapper/api/model/metadata.ex @@ -1,3 +1,3 @@ -defmodule Scrapper.Data.Api.Model.Match.Metadata do +defmodule Scrapper.Api.Model.Metadata do defstruct [:dataVersion, :matchId, :participants] end diff --git a/apps/scrapper/lib/scrapper/data/api/match/participant.ex b/apps/scrapper/lib/scrapper/api/model/participant.ex similarity index 98% rename from apps/scrapper/lib/scrapper/data/api/match/participant.ex rename to apps/scrapper/lib/scrapper/api/model/participant.ex index 1007aa0..bd4ec9a 100644 --- a/apps/scrapper/lib/scrapper/data/api/match/participant.ex +++ b/apps/scrapper/lib/scrapper/api/model/participant.ex @@ -1,4 +1,4 @@ -defmodule Scrapper.Data.Api.Model.Match.Participant do +defmodule Scrapper.Api.Model.Participant do # Enum.map(participant, fn {k,_v} -> ":#{k}" end) |> Enum.join(", ") defstruct [ :onMyWayPings, diff --git a/apps/scrapper/lib/scrapper/data/api/match/match_response.ex b/apps/scrapper/lib/scrapper/data/api/match/match_response.ex deleted file mode 100644 index d5b2929..0000000 --- a/apps/scrapper/lib/scrapper/data/api/match/match_response.ex +++ /dev/null @@ -1,6 +0,0 @@ -defmodule Scrapper.Data.Api.Model.Match.MatchResponse do - alias Scrapper.Data.Api.Model.Match.{Info, Metadata} - - defstruct metadata: %Metadata{}, - info: %Info{} -end diff --git a/apps/scrapper/lib/scrapper/data/storage/s3_match_storage.ex b/apps/scrapper/lib/scrapper/data/storage/s3_match_storage.ex deleted file mode 100644 index 3ac7be7..0000000 --- a/apps/scrapper/lib/scrapper/data/storage/s3_match_storage.ex +++ /dev/null @@ -1,11 +0,0 @@ -defmodule Scrapper.Data.Storage.S3MatchStorage do - @behaviour Scrapper.Data.Storage.MatchStorage - - def get_match(match_id) do - "" - end - - def store_match(match_id, match_data) do - :ok - end -end diff --git a/apps/scrapper/lib/scrapper/processor/match_processor.ex b/apps/scrapper/lib/scrapper/processor/match_processor.ex index 527b324..356b13a 100644 --- a/apps/scrapper/lib/scrapper/processor/match_processor.ex +++ b/apps/scrapper/lib/scrapper/processor/match_processor.ex @@ -20,8 +20,8 @@ defmodule Scrapper.Processor.MatchProcessor do ]}, concurrency: 1, rate_limiting: [ - interval: 1000 * 90, - allowed_messages: 5 + interval: 1000 * 60, + allowed_messages: 150 ] ], processors: [ @@ -35,20 +35,32 @@ defmodule Scrapper.Processor.MatchProcessor do @impl true def handle_message(_, message = %Broadway.Message{}, _) do match_id = message.data - IO.inspect(match_id) - match = Scrapper.Data.Api.MatchApi.get_match_by_id(match_id) + resp = Scrapper.Data.Api.MatchApi.get_match_by_id(match_id) + process_resp(resp, match_id) - match.metadata.participants - |> Enum.each(fn participant -> - Scrapper.Data.Api.MatchApi.get_matches_from_player(participant) - |> Enum.each(fn match_id -> - nil - Scrapper.Queue.MatchQueue.queue_match(match_id) - end) - end) - - IO.inspect(match.info.participants) message end + + def process_resp({:ok, raw_match}, match_id) do + decoded_match = Poison.decode!(raw_match, as: %Scrapper.Api.Model.MatchResponse{}) + Scrapper.Storage.S3MatchStorage.store_match(match_id, raw_match) + + decoded_match.metadata.participants + |> Enum.each(fn participant -> + case Scrapper.Data.Api.MatchApi.get_matches_from_player(participant) do + {:ok, matches} -> + matches + |> Enum.each(fn match_id -> + Scrapper.Queue.MatchQueue.queue_match(match_id) + end) + + {:err, code} -> + {:err, code} + end + end) + end + + def process_resp({:err, code}, match_id) do + end end diff --git a/apps/scrapper/lib/scrapper/processor/player_processor.ex b/apps/scrapper/lib/scrapper/processor/player_processor.ex index fc0dfe5..783fd74 100644 --- a/apps/scrapper/lib/scrapper/processor/player_processor.ex +++ b/apps/scrapper/lib/scrapper/processor/player_processor.ex @@ -20,13 +20,13 @@ defmodule Scrapper.Processor.PlayerProcessor do ]}, concurrency: 1, rate_limiting: [ - interval: 1000 * 90, - allowed_messages: 2 + interval: 1000 * 60, + allowed_messages: 5 ] ], processors: [ default: [ - concurrency: 2 + concurrency: 5 ] ] ) @@ -36,13 +36,20 @@ defmodule Scrapper.Processor.PlayerProcessor do def handle_message(_, message = %Broadway.Message{}, _) do puuid = message.data - IO.inspect(puuid) + resp = Scrapper.Data.Api.MatchApi.get_matches_from_player(puuid) - Scrapper.Data.Api.MatchApi.get_matches_from_player(puuid) - |> Enum.each(fn match_id -> - IO.inspect(match_id) - Scrapper.Queue.MatchQueue.queue_match(match_id) - end) + case resp do + {:ok, matches} -> + { + matches + |> Enum.each(fn match_id -> + Scrapper.Queue.MatchQueue.queue_match(match_id) + end) + } + + {:err, code} -> + {:err, code} + end message end diff --git a/apps/scrapper/lib/scrapper/queue/match_queue.ex b/apps/scrapper/lib/scrapper/queue/match_queue.ex index 29ab45b..0a266e8 100644 --- a/apps/scrapper/lib/scrapper/queue/match_queue.ex +++ b/apps/scrapper/lib/scrapper/queue/match_queue.ex @@ -18,7 +18,7 @@ defmodule Scrapper.Queue.MatchQueue do GenServer.call(__MODULE__, {:queue_match, match_id}) end - def handle_call({:queue_match, match_id}, from, %{:channel => channel} = state) do + def handle_call({:queue_match, match_id}, _from, %{:channel => channel} = state) do AMQP.Basic.publish(channel, "", "match", match_id) {:reply, nil, state} end diff --git a/apps/scrapper/lib/scrapper/data/storage/match_storage.ex b/apps/scrapper/lib/scrapper/storage/match_storage.ex similarity index 100% rename from apps/scrapper/lib/scrapper/data/storage/match_storage.ex rename to apps/scrapper/lib/scrapper/storage/match_storage.ex diff --git a/apps/scrapper/lib/scrapper/storage/s3_match_storage.ex b/apps/scrapper/lib/scrapper/storage/s3_match_storage.ex new file mode 100644 index 0000000..100d7ce --- /dev/null +++ b/apps/scrapper/lib/scrapper/storage/s3_match_storage.ex @@ -0,0 +1,33 @@ +defmodule Scrapper.Storage.S3MatchStorage do + require Logger + import SweetXml + + @behaviour Scrapper.Data.Storage.MatchStorage + + def get_match(match_id) do + "" + end + + @doc """ + + iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content" + """ + @spec store_match(String.t(), String.t()) :: none() + def store_match(match_id, match_data) do + File.write("/tmp/#{match_id}.json", match_data) + + url = + "/tmp/#{match_id}.json" + |> ExAws.S3.Upload.stream_file() + |> ExAws.S3.upload("matches", "#{match_id}.json") + |> ExAws.request!() + |> extract_s3_url_from_upload + + Logger.info("Stored match at #{url}") + + url + end + + defp extract_s3_url_from_upload(%{:body => %{:location => location}} = _s3_response), + do: location +end diff --git a/apps/scrapper/mix.exs b/apps/scrapper/mix.exs index 835fcc0..4e0bc1c 100644 --- a/apps/scrapper/mix.exs +++ b/apps/scrapper/mix.exs @@ -29,11 +29,12 @@ defmodule Scrapper.MixProject do {:httpoison, "~> 2.2"}, {:poison, "~> 5.0"}, {:ex_aws, "~> 2.1"}, - {:ex_aws_s3, "~> 2.0"}, + {:ex_aws_s3, "~> 2.5.3"}, {:hackney, "~> 1.9"}, {:sweet_xml, "~> 0.6"}, {:broadway_rabbitmq, "~> 0.7"}, - {:amqp, "~> 3.3"} + {:amqp, "~> 3.3"}, + {:sweet_xml, "~> 0.7.3"} # {:dep_from_hexpm, "~> 0.3.0"}, # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}, # {:sibling_app_in_umbrella, in_umbrella: true}