move storage and api to apps, create base analyzer

apps/storage/.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

apps/storage/.gitignore

@@ -0,0 +1,26 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+storage-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/

apps/storage/README.md

@@ -0,0 +1,21 @@
+# Storage
+
+**TODO: Add description**
+
+## Installation
+
+If [available in Hex](https://hex.pm/docs/publish), the package can be installed
+by adding `storage` to your list of dependencies in `mix.exs`:
+
+```elixir
+def deps do
+  [
+    {:storage, "~> 0.1.0"}
+  ]
+end
+```
+
+Documentation can be generated with [ExDoc](https://github.com/elixir-lang/ex_doc)
+and published on [HexDocs](https://hexdocs.pm). Once published, the docs can
+be found at <https://hexdocs.pm/storage>.
+

apps/storage/config/config.exs

@@ -0,0 +1,3 @@
+import Config
+
+import_config "#{config_env()}.exs"

apps/storage/config/dev.exs

@@ -0,0 +1,3 @@
+import Config
+
+import_config("libs/ex_aws_dev.exs")

apps/storage/config/libs/ex_aws_dev.exs

@@ -0,0 +1,17 @@
+import Config
+
+# config :ex_aws, :s3,
+#   scheme: "http://",
+#   host: System.get_env("EX_AWS_HOST"),
+#   port: System.get_env("EX_AWS_PORT"),
+#   access_key_id: System.get_env("EX_AWS_ACCESS_KEY"),
+#   secret_access_key: System.get_env("EX_AWS_SECRET_KEY")
+
+config :ex_aws,
+  access_key_id: "3zwMWl4RPCs8CHzhKmIX",
+  secret_access_key: "79B6LmryjJElkrIiHgDcfIxSpmvrLdVy75MyAJC2",
+  s3: [
+    scheme: "http://",
+    host: "localhost",
+    port: "9000"
+  ]

apps/storage/config/prod.exs

@@ -0,0 +1 @@
+import Config

apps/storage/lib/match_storage/match_storage.ex

@@ -0,0 +1,6 @@
+defmodule Storage.MatchStorage do
+  @callback get_match(String.t()) :: {:ok, Scrapper.Data.Match.t()} | {:error, :not_found}
+  @callback save_match(String.t(), Scrapper.Data.Match.t()) :: :ok
+  @callback list_matches() :: [map()]
+  @callback store_match(match_id :: String.t(), match :: map(), path :: String.t()) :: String.t()
+end

apps/storage/lib/match_storage/s3_match_storage.ex

@@ -0,0 +1,82 @@
+defmodule Storage.MatchStorage.S3MatchStorage do
+  require Logger
+  @behaviour Storage.MatchStorage
+
+  def get_match(match_id) do
+    ""
+  end
+
+  # check for to get all pages next_continuation_token
+  @impl true
+  def list_matches() do
+    {:ok, %{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} =
+      ExAws.S3.list_objects_v2("matches")
+      |> ExAws.request()
+
+    if next_continuation_token do
+      list_matches(contents, next_continuation_token)
+      # |> Enum.map(fn %{key: key} -> key end)
+    else
+      contents
+      # |> Enum.map(fn %{key: key} -> key end)
+    end
+  end
+
+  @spec list_matches(list(String.t()), String.t()) :: list(String.t())
+  defp list_matches(acc, continuation_token) do
+    resp =
+      {:ok,
+       %{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} =
+      ExAws.S3.list_objects_v2("matches", continuation_token: continuation_token)
+      |> ExAws.request()
+
+    if next_continuation_token == "" do
+      acc ++ contents
+    else
+      list_matches(acc ++ contents, next_continuation_token)
+    end
+  end
+
+  @doc """
+  iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content", "matches"
+  """
+  @impl true
+  @spec store_match(String.t(), String.t(), String.t()) :: none()
+  def store_match(match_id, match_data, bucket) do
+    File.write("/tmp/#{match_id}.json", match_data)
+
+    url =
+      "/tmp/#{match_id}.json"
+      |> ExAws.S3.Upload.stream_file()
+      |> ExAws.S3.upload(bucket, "#{match_id}.json")
+      |> ExAws.request!()
+      |> extract_s3_url_from_upload
+
+    Logger.info("Stored match at #{url}")
+
+    url
+  end
+
+  @doc """
+  iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content", "ranked" "14.9"
+  """
+  @impl true
+  @spec store_match(String.t(), String.t(), String.t(), String.t()) :: none()
+  def store_match(match_id, match_data, bucket, path) do
+    File.write("/tmp/#{match_id}.json", match_data)
+
+    url =
+      "/tmp/#{match_id}.json"
+      |> ExAws.S3.Upload.stream_file()
+      |> ExAws.S3.upload("#{bucket}/#{path}", "#{match_id}.json")
+      |> ExAws.request!()
+      |> extract_s3_url_from_upload
+
+    Logger.info("Stored match at #{url}")
+
+    url
+  end
+
+  defp extract_s3_url_from_upload(%{:body => %{:location => location}} = _s3_response),
+    do: location
+end

apps/storage/lib/storage.ex

@@ -0,0 +1,18 @@
+defmodule Storage do
+  @moduledoc """
+  Documentation for `Storage`.
+  """
+
+  @doc """
+  Hello world.
+
+  ## Examples
+
+      iex> Storage.hello()
+      :world
+
+  """
+  def hello do
+    :world
+  end
+end

apps/storage/mix.exs

@@ -0,0 +1,37 @@
+defmodule Storage.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :storage,
+      version: "0.1.0",
+      build_path: "../../_build",
+      config_path: "../../config/config.exs",
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.16",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications.
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  # Run "mix help deps" to learn about dependencies.
+  defp deps do
+    [
+      {:ex_aws, "~> 2.1"},
+      {:ex_aws_s3, "~> 2.5.3"},
+      {:hackney, "~> 1.9"},
+      {:sweet_xml, "~> 0.6"}
+      # {:dep_from_hexpm, "~> 0.3.0"},
+      # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"},
+      # {:sibling_app_in_umbrella, in_umbrella: true}
+    ]
+  end
+end

apps/storage/test/storage_test.exs

@@ -0,0 +1,8 @@
+defmodule StorageTest do
+  use ExUnit.Case
+  doctest Storage
+
+  test "greets the world" do
+    assert Storage.hello() == :world
+  end
+end

apps/storage/test/test_helper.exs

@@ -0,0 +1 @@
+ExUnit.start()