From 2f444e421867cced3a23b8ddf2da39ec12ead876 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lvaro?= Date: Sun, 5 May 2024 02:39:06 +0200 Subject: [PATCH] Add methods to S3 storage to list all matches --- .../lib/scrapper/storage/s3_match_storage.ex | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/apps/scrapper/lib/scrapper/storage/s3_match_storage.ex b/apps/scrapper/lib/scrapper/storage/s3_match_storage.ex index 100d7ce..9d9555a 100644 --- a/apps/scrapper/lib/scrapper/storage/s3_match_storage.ex +++ b/apps/scrapper/lib/scrapper/storage/s3_match_storage.ex @@ -8,8 +8,42 @@ defmodule Scrapper.Storage.S3MatchStorage do "" end - @doc """ + # check for to get all pages next_continuation_token + def list_matches() do + {:ok, %{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} = + ExAws.S3.list_objects_v2("matches") + |> ExAws.request() + if next_continuation_token do + list_matches(contents, next_continuation_token) + # |> Enum.map(fn %{key: key} -> key end) + else + contents + # |> Enum.map(fn %{key: key} -> key end) + end + end + + @spec list_matches(list(String.t()), String.t()) :: list(String.t()) + def list_matches(acc, continuation_token) do + resp = + {:ok, + %{:body => %{:contents => contents, next_continuation_token: next_continuation_token}}} = + ExAws.S3.list_objects_v2("matches", continuation_token: continuation_token) + |> ExAws.request() + + if next_continuation_token == "" do + acc ++ contents + else + list_matches(acc ++ contents, next_continuation_token) + end + end + + def download_match(destination_path, url) do + ExAws.S3.download_file(url, destination_path) + |> ExAws.request() + end + + @doc """ iex> Scrapper.Storage.S3MatchStorage.store_match "1", "content" """ @spec store_match(String.t(), String.t()) :: none()