From e8478da3dafa5fe610ec2db14bcb5e5c1c23c1a6 Mon Sep 17 00:00:00 2001 From: Correl Roush Date: Thu, 18 Jan 2018 14:15:29 -0500 Subject: [PATCH] Initial commit --- .gitignore | 3 + LICENSE | 19 ++++ README.md | 67 +++++++++++++ elm-package.json | 19 ++++ src/Paginated.elm | 202 +++++++++++++++++++++++++++++++++++++++ src/Paginated/Util.elm | 35 +++++++ tests/PaginatedTests.elm | 33 +++++++ tests/elm-package.json | 20 ++++ 8 files changed, 398 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 elm-package.json create mode 100644 src/Paginated.elm create mode 100644 src/Paginated/Util.elm create mode 100644 tests/PaginatedTests.elm create mode 100644 tests/elm-package.json diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75b0e64 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +elm.js +elm-stuff/ +node_modules/ \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f4f44af --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2018 Correl Roush + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f0d1f76 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +A library for fetching data from paginated JSON REST APIs. + +# Motivation + +This library is built to handle fetching data from a JSON REST API +that paginates its results. It inspects response headers, and will +prepare subsequent requests when additional pages of results remain. + +Given a JSON decoder for the data type returned in the paginated +collection, it will return a data structure containing the results +fetched, and a new request object for fetching additional records from +the next page if more are available. + +It is expected that the paginated REST API will provide a link for the +next page of results in the `Link` header, e.g.: + + Link: ; rel="next" + +Absence of a link with `rel="next"` in the response headers is assumed +to mean the final page of results has been reached. + +# Example Usage + + import Http + import Json.Decode exposing (string) + import Paginated exposing (Response(..)) + + type alias Model = + { results : Maybe (Paginated.Response String) } + + type Msg + = Search + | Results (Result Http.Error (Paginated.Response String)) + + update : Msg -> Model -> ( Model, Cmd Msg ) + update msg model = + case msg of + Search -> + ( model, doSearch ) + + Results (Ok response) -> + case response of + Partial request results -> + ( { model + | results = + Maybe.map (\x -> Paginated.update x response) + model.results + } + , Paginated.send Results request + ) + + Complete results -> + ( { model + | results = + Maybe.map (\x -> Paginated.update x response) + model.results + } + , Cmd.none + ) + + Results (Err _) -> + ( model, Cmd.none ) + + doSearch : Cmd Msg + doSearch = + Paginated.send Results <| + Paginated.get "http://example.com/items" string diff --git a/elm-package.json b/elm-package.json new file mode 100644 index 0000000..1822a48 --- /dev/null +++ b/elm-package.json @@ -0,0 +1,19 @@ +{ + "version": "1.0.0", + "summary": "A library for fetching data from paginated JSON REST APIs.", + "repository": "https://github.com/correl/elm-paginated.git", + "license": "BSD3", + "source-directories": [ + "src" + ], + "exposed-modules": [ + "Paginated" + ], + "dependencies": { + "elm-community/maybe-extra": "4.0.0 <= v < 5.0.0", + "elm-lang/core": "5.1.1 <= v < 6.0.0", + "elm-lang/html": "2.0.0 <= v < 3.0.0", + "elm-lang/http": "1.0.0 <= v < 2.0.0" + }, + "elm-version": "0.18.0 <= v < 0.19.0" +} diff --git a/src/Paginated.elm b/src/Paginated.elm new file mode 100644 index 0000000..01bb4e3 --- /dev/null +++ b/src/Paginated.elm @@ -0,0 +1,202 @@ +module Paginated + exposing + ( Request + , RequestOptions + , Response(..) + , request + , get + , post + , send + , update + , httpRequest + ) + +{-| A library for Facilitates fetching data from a paginated JSON API. + +# Requests and Responses + +@docs Request, Response, get, post + + +## Custom requests + +@docs RequestOptions, request + + +## Converting + +@docs httpRequest + + +# Sending requests + +@docs send + + +# Handling responses + +@docs Response, update + +-} + +import Dict exposing (Dict) +import Http +import Json.Decode exposing (Decoder) +import Maybe.Extra +import Paginated.Util +import Time + + +{-| Describes an API request. +-} +type alias RequestOptions a = + { method : String + , headers : List Http.Header + , url : String + , body : Http.Body + , decoder : Decoder a + , timeout : Maybe Time.Time + , withCredentials : Bool + } + + +{-| Encapsulates an API request for a list of items of type `a`. +-} +type Request a + = Request (RequestOptions a) + + +{-| Describes an API response. + +A response may either be Partial (there are more pages of results yet +to be fetched), or Complete (all records have been fetched). The +response includes all of the items fetched in order. + +-} +type Response a + = Partial (Request a) (List a) + | Complete (List a) + + +{-| Create a custom request, allowing the specification of HTTP +headers and other options. For example: + + Paginated.request + { method = "GET" + , headers = [Http.header "Private-Token" "XXXXXXXXXXXXXXXX"] + , url = url + , body = Http.emptyBody + , decoder = decoder + , timeout = Nothing + , withCredentials = False + } + +-} +request : RequestOptions a -> Request a +request = + Request + + +{-| Build a GET request. +-} +get : String -> Decoder a -> Request a +get url decoder = + request + { method = "GET" + , headers = [] + , url = url + , body = Http.emptyBody + , decoder = decoder + , timeout = Nothing + , withCredentials = False + } + + +{-| Build a POST request. +-} +post : String -> Http.Body -> Decoder a -> Request a +post url body decoder = + request + { method = "POST" + , headers = [] + , url = url + , body = body + , decoder = decoder + , timeout = Nothing + , withCredentials = False + } + + +{-| Send a `Request`. +-} +send : + (Result Http.Error (Response a) -> msg) + -> Request a + -> Cmd msg +send resultToMessage request = + Http.send resultToMessage <| + httpRequest request + + +{-| Append two paginated responses, collecting the results within. +-} +update : Response a -> Response a -> Response a +update old new = + case ( old, new ) of + ( Complete items, _ ) -> + Complete items + + ( Partial _ oldItems, Complete newItems ) -> + Complete (oldItems ++ newItems) + + ( Partial _ oldItems, Partial request newItems ) -> + Partial request (oldItems ++ newItems) + + +{-| Convert a `Request` to a `Http.Request` that can then be sent via +`Http.send`. +-} +httpRequest : Request a -> Http.Request (Response a) +httpRequest (Request options) = + Http.request + { method = options.method + , headers = options.headers + , url = options.url + , body = options.body + , expect = expect options + , timeout = options.timeout + , withCredentials = options.withCredentials + } + + +expect : RequestOptions a -> Http.Expect (Response a) +expect options = + Http.expectStringResponse (fromResponse options) + + +fromResponse : + RequestOptions a + -> Http.Response String + -> Result String (Response a) +fromResponse options response = + let + items : Result String (List a) + items = + Json.Decode.decodeString + (Json.Decode.list options.decoder) + response.body + + nextPage = + Dict.get "Link" response.headers + |> Maybe.map Paginated.Util.links + |> Maybe.andThen (Dict.get "next") + in + case nextPage of + Nothing -> + Result.map Complete items + + Just url -> + Result.map + (Partial (request { options | url = url })) + items + diff --git a/src/Paginated/Util.elm b/src/Paginated/Util.elm new file mode 100644 index 0000000..e576681 --- /dev/null +++ b/src/Paginated/Util.elm @@ -0,0 +1,35 @@ +module Paginated.Util exposing (links) + +import Dict exposing (Dict) +import Maybe.Extra +import Regex + + +{-| Parse an HTTP Link header into a dictionary. For example, to look +for a link to additional results in an API response, you could do the +following: + + Dict.get "Link" response.headers + |> Maybe.map links + |> Maybe.andThen (Dict.get "next") + +-} +links : String -> Dict String String +links s = + let + toTuples xs = + case xs of + [ Just a, Just b ] -> + Just ( b, a ) + + _ -> + Nothing + in + Regex.find + Regex.All + (Regex.regex "<(.*?)>; rel=\"(.*?)\"") + s + |> List.map .submatches + |> List.map toTuples + |> Maybe.Extra.values + |> Dict.fromList diff --git a/tests/PaginatedTests.elm b/tests/PaginatedTests.elm new file mode 100644 index 0000000..30b3313 --- /dev/null +++ b/tests/PaginatedTests.elm @@ -0,0 +1,33 @@ +module PaginatedTests exposing (..) + +import Dict +import Expect +import Paginated +import Paginated.Util +import Test exposing (..) + + +suite : Test +suite = + describe "Paginated" + [ test "Parse links" <| + \() -> + let + header = + String.join ", " + [ "; rel=\"prev\"" + , "; rel=\"next\"" + , "; rel=\"first\"" + , "; rel=\"last\"" + ] + + expected = + Dict.fromList + [ ( "prev", "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3" ) + , ( "next", "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3" ) + , ( "first", "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=1&per_page=3" ) + , ( "last", "https://gitlab.example.com/api/v4/projects/8/issues/8/notes?page=3&per_page=3" ) + ] + in + Expect.equalDicts expected (Paginated.Util.links header) + ] diff --git a/tests/elm-package.json b/tests/elm-package.json new file mode 100644 index 0000000..ba10ad5 --- /dev/null +++ b/tests/elm-package.json @@ -0,0 +1,20 @@ +{ + "version": "1.0.0", + "summary": "Test Suites", + "repository": "https://github.com/correl/elm-paginated.git", + "license": "BSD3", + "source-directories": [ + "../src", + "." + ], + "exposed-modules": [], + "dependencies": { + "eeue56/elm-html-test": "5.1.2 <= v < 6.0.0", + "elm-community/elm-test": "4.0.0 <= v < 5.0.0", + "elm-community/maybe-extra": "4.0.0 <= v < 5.0.0", + "elm-lang/core": "5.1.1 <= v < 6.0.0", + "elm-lang/html": "2.0.0 <= v < 3.0.0", + "elm-lang/http": "1.0.0 <= v < 2.0.0" + }, + "elm-version": "0.18.0 <= v < 0.19.0" +}