Paginate search results

pyproject.toml

@@ -19,7 +19,7 @@ black = "^21.6b0"
 mypy = "^0.910"
 
 [tool.poetry.scripts]
-tutor = 'tutor.__main__:cli'
+tutor = 'tutor.cli:main'
 
 [build-system]
 requires = ["poetry>=0.12"]

tutor/__main__.py

@@ -1,117 +0,0 @@
-import json
-import logging
-
-import aiosqlite
-import click
-import httpx
-import humanize
-import tornado.ioloop
-import tornado.web
-
-import tutor.csvimport
-import tutor.database
-import tutor.scryfall
-import tutor.server
-
-
-@click.group()
-@click.option(
-    "--database",
-    envvar="TUTOR_DATABASE",
-    type=click.Path(dir_okay=False),
-    required=True,
-)
-@click.option(
-    "--log-level",
-    type=click.Choice(
-        ["debug", "info", "warn", "error"],
-        case_sensitive=False,
-    ),
-    default="warn",
-)
-@click.pass_context
-def cli(ctx, database, log_level):
-    logging.basicConfig(
-        level={
-            "debug": logging.DEBUG,
-            "info": logging.INFO,
-            "warn": logging.WARN,
-            "error": logging.ERROR,
-        }.get(log_level.lower())
-    )
-    ctx.ensure_object(dict)
-    ctx.obj["database"] = database
-
-
-@cli.command()
-@click.option("--port", type=int, envvar="TUTOR_PORT", default=8888)
-@click.option("--static", envvar="TUTOR_STATIC", type=click.Path(file_okay=False))
-@click.option("--debug", is_flag=True)
-@click.pass_context
-def server(ctx, port, static, debug):
-    app = tutor.server.make_app(
-        {
-            **ctx.obj,
-            "static": static,
-            "debug": debug,
-        }
-    )
-    app.listen(port)
-    tornado.ioloop.IOLoop.current().start()
-
-
-@cli.command("import")
-@click.argument("filename", type=click.Path(dir_okay=False))
-@click.pass_context
-def import_cards(ctx, filename):
-    tornado.ioloop.IOLoop.current().run_sync(
-        lambda: tutor.csvimport.load(ctx.obj, filename)
-    )
-
-
-@cli.command("update_scryfall")
-@click.option("--filename", type=click.Path(dir_okay=False))
-@click.pass_context
-def update_scryfall(ctx, filename):
-    if filename:
-        with open(filename) as f:
-            cards = json.loads(f.read())
-    else:
-        response = httpx.get("https://api.scryfall.com/bulk-data/oracle_cards")
-        info = response.json()
-
-        buffer = b""
-        with httpx.stream("GET", info["download_uri"]) as response:
-            downloaded = response.num_bytes_downloaded
-            total = int(response.headers["Content-Length"])
-            with click.progressbar(
-                length=total,
-                label=f"Downloading {humanize.naturalsize(total)}"
-                " of card data from Scryfall",
-            ) as bar:
-                for chunk in response.iter_bytes():
-                    buffer += chunk
-                    bar.update(response.num_bytes_downloaded - downloaded)
-                    downloaded = response.num_bytes_downloaded
-        cards = json.loads(buffer)
-
-    async def import_cards():
-        async with aiosqlite.connect(ctx.obj["database"]) as db:
-            with click.progressbar(
-                cards,
-                label=f"Importing {humanize.intcomma(len(cards))} cards",
-            ) as bar:
-                for card_object in bar:
-                    await tutor.database.store_card(
-                        db, tutor.scryfall.to_card(card_object)
-                    )
-                    await tutor.database.store_set(
-                        db, card_object["set"].upper(), card_object["set_name"]
-                    )
-            await db.commit()
-
-    tornado.ioloop.IOLoop.current().run_sync(import_cards)
-
-
-if __name__ == "__main__":
-    cli()

tutor/database.py

@@ -93,6 +93,7 @@ async def advanced_search(
     db: aiosqlite.Connection,
     search: tutor.search.Search,
     limit: int = 10,
+    offset: int = 0,
     in_collection: typing.Optional[bool] = None,
 ) -> typing.List[tutor.models.Card]:
     db.row_factory = aiosqlite.Row
@@ -146,7 +147,7 @@ async def advanced_search(
             " ".join(joins),
             "WHERE" if constraints else "",
             " AND ".join(constraints),
-            f"LIMIT {limit}",
+            f"LIMIT {offset},{limit}",
         ]
     )
     cursor = await db.execute(query, params)

tutor/server.py

@@ -1,4 +1,5 @@
 import json
+import urllib.parse
 
 import aiosqlite
 import tornado.web
@@ -7,21 +8,57 @@ import tutor.database
 import tutor.models
 import tutor.search
 
+
+def update_args(url: str, **qargs) -> str:
+    parts = urllib.parse.urlsplit(url)
+    return urllib.parse.urlunsplit(
+        (
+            parts.scheme,
+            parts.netloc,
+            parts.path,
+            urllib.parse.urlencode(
+                [
+                    (k, v)
+                    for k, v in urllib.parse.parse_qsl(parts.query)
+                    if k not in qargs.keys()
+                ]
+                + list(qargs.items())
+            ),
+            parts.fragment,
+        )
+    )
+
+
 class SearchHandler(tornado.web.RequestHandler):
+    def set_links(self, **links) -> None:
+        self.set_header(
+            "Link",
+            ", ".join([f'<{url}>; rel="{rel}"' for rel, url in links.items()]),
+        )
+
     async def get(self) -> None:
         async with aiosqlite.connect(self.application.settings["database"]) as db:
-            name = self.get_argument("name", None)
+            query = self.get_argument("q", "")
             in_collection = self.get_argument("in_collection", None)
+            page = max(1, int(self.get_argument("page", 1)))
             limit = int(self.get_argument("limit", 10))
-            search = tutor.search.search.parse(name)
+            search = tutor.search.search.parse(query)
             cards = await tutor.database.advanced_search(
                 db,
                 search,
-                limit=limit,
+                limit=limit + 1,
+                offset=limit * (page - 1),
                 in_collection=in_collection,
             )
+        has_more = cards and len(cards) > limit
         self.set_header("Content-Type", "application/json")
         self.set_header("Access-Control-Allow-Origin", "*")
+        links = {}
+        if page > 1:
+            links["prev"] = update_args(self.request.full_url(), page=page - 1)
+        if has_more:
+            links["next"] = update_args(self.request.full_url(), page=page + 1)
+        self.set_links(**links)
         self.write(
             json.dumps(
                 [
@@ -35,7 +72,7 @@ class SearchHandler(tornado.web.RequestHandler):
                             card.color_identity
                         ),
                     }
-                    for card in cards
+                    for card in cards[:limit]
                 ]
             )
         )

www/elm.json

@@ -12,7 +12,9 @@
             "elm/html": "1.0.0",
             "elm/http": "2.0.0",
             "elm/json": "1.1.3",
+            "elm/regex": "1.0.0",
             "elm/url": "1.0.0",
+            "elm-community/maybe-extra": "5.2.0",
             "mdgriffith/elm-ui": "1.1.8"
         },
         "indirect": {

www/public/index.html

@@ -3,6 +3,7 @@
         <title>Bulk Tagging Dashboard</title>
         <script type="text/javascript" src="elm.js"></script>
     </head>
+    <meta charset="utf-8"/>
     <body>
         <script type="text/javascript">
          var app = Elm.App.init({flags: {}});

www/src/App.elm

@@ -1,22 +1,33 @@
 module App exposing (main)
 
 import Browser
+import Browser.Dom
+import Browser.Events
+import Dict
 import Element as E
 import Element.Background as Background
 import Element.Border as Border
 import Element.Font as Font
 import Element.Input as Input
-import Html
 import Html.Events
 import Http
 import Json.Decode
 import Json.Decode.Pipeline as JDP
+import Maybe.Extra
+import Regex
+import Task
 import Url
 import Url.Builder
 
 
+type alias Window =
+    { width : Int
+    , height : Int
+    }
+
+
 type alias Criteria =
-    { name : String
+    { query : String
     , ownedOnly : Bool
     }
 
@@ -30,17 +41,20 @@ type alias Card =
 
 
 type alias Model =
-    { criteria : Criteria
+    { viewport : Window
-    , cards : List Card
+    , criteria : Criteria
+    , cardPage : Maybe (ResultPage Card)
     }
 
 
 type Msg
     = UrlChanged Url.Url
+    | ViewportChanged Window
     | LinkClicked Browser.UrlRequest
     | UpdateCriteria CriteriaMsg
     | Search
-    | FoundCards (Result Http.Error (List Card))
+    | GetPage Url.Url
+    | FoundCards (Result Http.Error (ResultPage Card))
 
 
 type CriteriaMsg
@@ -48,13 +62,81 @@ type CriteriaMsg
     | UpdateOwnedOnly Bool
 
 
+type alias ResultPage a =
+    { prev : Maybe Url.Url
+    , next : Maybe Url.Url
+    , values : List a
+    }
+
+
+expectPaginatedJson : (Result Http.Error (ResultPage value) -> msg) -> Json.Decode.Decoder value -> Http.Expect msg
+expectPaginatedJson toMsg decoder =
+    let
+        linkPattern : Regex.Regex
+        linkPattern =
+            Regex.fromString "<(.*?)>; rel=\"(.*?)\""
+                |> Maybe.withDefault Regex.never
+
+        links : String -> Dict.Dict String String
+        links s =
+            let
+                toTuples xs =
+                    case xs of
+                        [ Just a, Just b ] ->
+                            Just ( b, a )
+
+                        _ ->
+                            Nothing
+            in
+            Regex.find linkPattern s
+                |> List.map .submatches
+                |> List.map toTuples
+                |> Maybe.Extra.values
+                |> Dict.fromList
+    in
+    Http.expectStringResponse toMsg <|
+        \response ->
+            case response of
+                Http.BadUrl_ url ->
+                    Err (Http.BadUrl url)
+
+                Http.Timeout_ ->
+                    Err Http.Timeout
+
+                Http.NetworkError_ ->
+                    Err Http.NetworkError
+
+                Http.BadStatus_ metadata _ ->
+                    Err (Http.BadStatus metadata.statusCode)
+
+                Http.GoodStatus_ metadata body ->
+                    case Json.Decode.decodeString (Json.Decode.list decoder) body of
+                        Ok values ->
+                            Ok
+                                { prev =
+                                    Dict.get "link" (Debug.log "headers" metadata.headers)
+                                        |> Maybe.map links
+                                        |> Maybe.andThen (Dict.get "prev")
+                                        |> Maybe.andThen Url.fromString
+                                , next =
+                                    Dict.get "link" metadata.headers
+                                        |> Maybe.map links
+                                        |> Maybe.andThen (Dict.get "next")
+                                        |> Maybe.andThen Url.fromString
+                                , values = values
+                                }
+
+                        Err err ->
+                            Err (Http.BadBody (Json.Decode.errorToString err))
+
+
 search : Criteria -> Cmd Msg
 search criteria =
     Http.get
         { url =
             Url.Builder.absolute
                 [ "search" ]
-                [ Url.Builder.string "name" criteria.name
+                [ Url.Builder.string "q" criteria.query
                 , Url.Builder.string "in_collection"
                     (if criteria.ownedOnly then
                         "yes"
@@ -62,9 +144,17 @@ search criteria =
                      else
                         ""
                     )
-                , Url.Builder.int "limit" 20
+                , Url.Builder.int "limit" 18
                 ]
-        , expect = Http.expectJson FoundCards (Json.Decode.list decodeCard)
+        , expect = expectPaginatedJson FoundCards decodeCard
+        }
+
+
+loadPage : Url.Url -> Cmd Msg
+loadPage url =
+    Http.get
+        { url = Url.toString url
+        , expect = expectPaginatedJson FoundCards decodeCard
         }
 
 
@@ -78,29 +168,55 @@ decodeCard =
 
 
 init : Json.Decode.Value -> url -> key -> ( Model, Cmd Msg )
-init flag url key =
+init _ _ _ =
     let
         criteria =
-            { name = "", ownedOnly = True }
+            { query = "", ownedOnly = True }
     in
-    ( { criteria = criteria
+    ( { viewport = { width = 1280, height = 720 }
-      , cards = []
+      , criteria = criteria
+      , cardPage = Nothing
       }
-    , search criteria
+    , Cmd.batch
+        [ search criteria
+        , Task.perform
+            (\x ->
+                ViewportChanged
+                    { width = floor x.viewport.width
+                    , height = floor x.viewport.height
+                    }
+            )
+            Browser.Dom.getViewport
+        ]
     )
 
 
+updateCriteria : CriteriaMsg -> Criteria -> Criteria
 updateCriteria msg model =
     case msg of
         UpdateName text ->
-            { model | name = text }
+            { model | query = text }
 
         UpdateOwnedOnly value ->
             { model | ownedOnly = value }
 
 
+updateCardPage cardpage newCards =
+    { cardpage | cards = newCards }
+
+
+update : Msg -> Model -> ( Model, Cmd Msg )
 update msg model =
     case msg of
+        UrlChanged _ ->
+            ( model, Cmd.none )
+
+        LinkClicked _ ->
+            ( model, Cmd.none )
+
+        ViewportChanged viewport ->
+            ( { model | viewport = viewport }, Cmd.none )
+
         UpdateCriteria criteriaMsg ->
             ( { model | criteria = updateCriteria criteriaMsg model.criteria }
             , Cmd.none
@@ -109,10 +225,13 @@ update msg model =
         Search ->
             ( model, search model.criteria )
 
-        FoundCards (Ok cards) ->
+        GetPage url ->
-            ( { model | cards = cards }, Cmd.none )
+            ( model, loadPage url )
 
-        _ ->
+        FoundCards (Ok cardPage) ->
+            ( { model | cardPage = Just cardPage }, Cmd.none )
+
+        FoundCards (Err _) ->
             ( model, Cmd.none )
 
 
@@ -133,18 +252,70 @@ colors =
     }
 
 
-viewCard model =
+viewCardBrowser model =
-    E.column []
+    let
-        [ E.image [ E.height <| E.px 300 ]
+        cardWidth =
-            { src =
+            -- 50% of the Scryfall border_crop image width
-                Url.Builder.crossOrigin "https://api.scryfall.com"
+            240
-                    [ "cards", model.scryfallId ]
+
-                    [ Url.Builder.string "format" "image"
+        cardSpacing =
-                    , Url.Builder.string "version" "border_crop"
+            10
+
+        navigationButtonWidth =
+            50
+
+        cardColumns =
+            -- Either 3, 6, or 9, based on viewport width
+            let
+                availableColumns =
+                    (model.viewport.width - (2 * navigationButtonWidth))
+                        // (cardWidth + cardSpacing)
+            in
+            (max 3 >> min 9) (availableColumns // 3 * 3)
+
+        viewCard cardModel =
+            E.column []
+                [ E.image [ E.width (E.px cardWidth) ]
+                    { src =
+                        Url.Builder.crossOrigin "https://api.scryfall.com"
+                            [ "cards", cardModel.scryfallId ]
+                            [ Url.Builder.string "format" "image"
+                            , Url.Builder.string "version" "border_crop"
+                            ]
+                    , description = cardModel.name
+                    }
+                ]
+
+        navButton text maybeUrl =
+            case maybeUrl of
+                Just url ->
+                    Input.button
+                        [ E.height E.fill
+                        , E.width (E.px navigationButtonWidth)
+                        , Background.color colors.primary
+                        , Font.color colors.text
+                        , Font.center
+                        ]
+                        { label = E.text text, onPress = Just (GetPage url) }
+
+                Nothing ->
+                    E.el [ E.width (E.px navigationButtonWidth) ] E.none
+    in
+    case model.cardPage of
+        Nothing ->
+            E.none
+
+        Just cardPage ->
+            E.row [ E.width E.fill, E.centerX ] <|
+                [ navButton "←" cardPage.prev
+                , E.wrappedRow
+                    [ E.width (E.px (cardColumns * (cardWidth + cardSpacing)))
+                    , E.centerX
+                    , E.spacing cardSpacing
                     ]
-            , description = model.name
+                    (List.map viewCard cardPage.values)
-            }
+                , navButton "→" cardPage.next
-        ]
+                ]
 
 
 onEnter : msg -> E.Attribute msg
@@ -168,7 +339,7 @@ view model =
     { title = "Tutor"
     , body =
         [ E.layout [ Background.color colors.background ] <|
-            E.column []
+            E.column [ E.width E.fill ]
                 [ E.row [ E.spacing 10 ]
                     [ Input.text
                         [ onEnter Search
@@ -176,7 +347,7 @@ view model =
                         , Font.color colors.text
                         ]
                         { onChange = UpdateCriteria << UpdateName
-                        , text = model.criteria.name
+                        , text = model.criteria.query
                         , placeholder = Nothing
                         , label = Input.labelHidden "Search Input"
                         }
@@ -196,7 +367,7 @@ view model =
                         , label = Input.labelRight [ Font.color colors.text ] (E.text "Owned only?")
                         }
                     ]
-                , E.wrappedRow [ E.spacing 5, E.padding 30 ] <| List.map viewCard model.cards
+                , viewCardBrowser model
                 ]
         ]
     }
@@ -212,5 +383,7 @@ main =
         , subscriptions =
             \_ ->
                 Sub.batch
-                    []
+                    [ Browser.Events.onResize
+                        (\w h -> ViewportChanged { width = w, height = h })
+                    ]
         }