Paginate search results
This commit is contained in:
parent
f987799205
commit
e3a004df20
7 changed files with 252 additions and 155 deletions
|
|
@ -19,7 +19,7 @@ black = "^21.6b0"
|
|||
mypy = "^0.910"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
tutor = 'tutor.__main__:cli'
|
||||
tutor = 'tutor.cli:main'
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry>=0.12"]
|
||||
|
|
|
|||
|
|
@ -1,117 +0,0 @@
|
|||
import json
|
||||
import logging
|
||||
|
||||
import aiosqlite
|
||||
import click
|
||||
import httpx
|
||||
import humanize
|
||||
import tornado.ioloop
|
||||
import tornado.web
|
||||
|
||||
import tutor.csvimport
|
||||
import tutor.database
|
||||
import tutor.scryfall
|
||||
import tutor.server
|
||||
|
||||
|
||||
@click.group()
|
||||
@click.option(
|
||||
"--database",
|
||||
envvar="TUTOR_DATABASE",
|
||||
type=click.Path(dir_okay=False),
|
||||
required=True,
|
||||
)
|
||||
@click.option(
|
||||
"--log-level",
|
||||
type=click.Choice(
|
||||
["debug", "info", "warn", "error"],
|
||||
case_sensitive=False,
|
||||
),
|
||||
default="warn",
|
||||
)
|
||||
@click.pass_context
|
||||
def cli(ctx, database, log_level):
|
||||
logging.basicConfig(
|
||||
level={
|
||||
"debug": logging.DEBUG,
|
||||
"info": logging.INFO,
|
||||
"warn": logging.WARN,
|
||||
"error": logging.ERROR,
|
||||
}.get(log_level.lower())
|
||||
)
|
||||
ctx.ensure_object(dict)
|
||||
ctx.obj["database"] = database
|
||||
|
||||
|
||||
@cli.command()
|
||||
@click.option("--port", type=int, envvar="TUTOR_PORT", default=8888)
|
||||
@click.option("--static", envvar="TUTOR_STATIC", type=click.Path(file_okay=False))
|
||||
@click.option("--debug", is_flag=True)
|
||||
@click.pass_context
|
||||
def server(ctx, port, static, debug):
|
||||
app = tutor.server.make_app(
|
||||
{
|
||||
**ctx.obj,
|
||||
"static": static,
|
||||
"debug": debug,
|
||||
}
|
||||
)
|
||||
app.listen(port)
|
||||
tornado.ioloop.IOLoop.current().start()
|
||||
|
||||
|
||||
@cli.command("import")
|
||||
@click.argument("filename", type=click.Path(dir_okay=False))
|
||||
@click.pass_context
|
||||
def import_cards(ctx, filename):
|
||||
tornado.ioloop.IOLoop.current().run_sync(
|
||||
lambda: tutor.csvimport.load(ctx.obj, filename)
|
||||
)
|
||||
|
||||
|
||||
@cli.command("update_scryfall")
|
||||
@click.option("--filename", type=click.Path(dir_okay=False))
|
||||
@click.pass_context
|
||||
def update_scryfall(ctx, filename):
|
||||
if filename:
|
||||
with open(filename) as f:
|
||||
cards = json.loads(f.read())
|
||||
else:
|
||||
response = httpx.get("https://api.scryfall.com/bulk-data/oracle_cards")
|
||||
info = response.json()
|
||||
|
||||
buffer = b""
|
||||
with httpx.stream("GET", info["download_uri"]) as response:
|
||||
downloaded = response.num_bytes_downloaded
|
||||
total = int(response.headers["Content-Length"])
|
||||
with click.progressbar(
|
||||
length=total,
|
||||
label=f"Downloading {humanize.naturalsize(total)}"
|
||||
" of card data from Scryfall",
|
||||
) as bar:
|
||||
for chunk in response.iter_bytes():
|
||||
buffer += chunk
|
||||
bar.update(response.num_bytes_downloaded - downloaded)
|
||||
downloaded = response.num_bytes_downloaded
|
||||
cards = json.loads(buffer)
|
||||
|
||||
async def import_cards():
|
||||
async with aiosqlite.connect(ctx.obj["database"]) as db:
|
||||
with click.progressbar(
|
||||
cards,
|
||||
label=f"Importing {humanize.intcomma(len(cards))} cards",
|
||||
) as bar:
|
||||
for card_object in bar:
|
||||
await tutor.database.store_card(
|
||||
db, tutor.scryfall.to_card(card_object)
|
||||
)
|
||||
await tutor.database.store_set(
|
||||
db, card_object["set"].upper(), card_object["set_name"]
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
tornado.ioloop.IOLoop.current().run_sync(import_cards)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
cli()
|
||||
|
|
@ -93,6 +93,7 @@ async def advanced_search(
|
|||
db: aiosqlite.Connection,
|
||||
search: tutor.search.Search,
|
||||
limit: int = 10,
|
||||
offset: int = 0,
|
||||
in_collection: typing.Optional[bool] = None,
|
||||
) -> typing.List[tutor.models.Card]:
|
||||
db.row_factory = aiosqlite.Row
|
||||
|
|
@ -146,7 +147,7 @@ async def advanced_search(
|
|||
" ".join(joins),
|
||||
"WHERE" if constraints else "",
|
||||
" AND ".join(constraints),
|
||||
f"LIMIT {limit}",
|
||||
f"LIMIT {offset},{limit}",
|
||||
]
|
||||
)
|
||||
cursor = await db.execute(query, params)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import json
|
||||
import urllib.parse
|
||||
|
||||
import aiosqlite
|
||||
import tornado.web
|
||||
|
|
@ -7,21 +8,57 @@ import tutor.database
|
|||
import tutor.models
|
||||
import tutor.search
|
||||
|
||||
|
||||
def update_args(url: str, **qargs) -> str:
|
||||
parts = urllib.parse.urlsplit(url)
|
||||
return urllib.parse.urlunsplit(
|
||||
(
|
||||
parts.scheme,
|
||||
parts.netloc,
|
||||
parts.path,
|
||||
urllib.parse.urlencode(
|
||||
[
|
||||
(k, v)
|
||||
for k, v in urllib.parse.parse_qsl(parts.query)
|
||||
if k not in qargs.keys()
|
||||
]
|
||||
+ list(qargs.items())
|
||||
),
|
||||
parts.fragment,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class SearchHandler(tornado.web.RequestHandler):
|
||||
def set_links(self, **links) -> None:
|
||||
self.set_header(
|
||||
"Link",
|
||||
", ".join([f'<{url}>; rel="{rel}"' for rel, url in links.items()]),
|
||||
)
|
||||
|
||||
async def get(self) -> None:
|
||||
async with aiosqlite.connect(self.application.settings["database"]) as db:
|
||||
name = self.get_argument("name", None)
|
||||
query = self.get_argument("q", "")
|
||||
in_collection = self.get_argument("in_collection", None)
|
||||
page = max(1, int(self.get_argument("page", 1)))
|
||||
limit = int(self.get_argument("limit", 10))
|
||||
search = tutor.search.search.parse(name)
|
||||
search = tutor.search.search.parse(query)
|
||||
cards = await tutor.database.advanced_search(
|
||||
db,
|
||||
search,
|
||||
limit=limit,
|
||||
limit=limit + 1,
|
||||
offset=limit * (page - 1),
|
||||
in_collection=in_collection,
|
||||
)
|
||||
has_more = cards and len(cards) > limit
|
||||
self.set_header("Content-Type", "application/json")
|
||||
self.set_header("Access-Control-Allow-Origin", "*")
|
||||
links = {}
|
||||
if page > 1:
|
||||
links["prev"] = update_args(self.request.full_url(), page=page - 1)
|
||||
if has_more:
|
||||
links["next"] = update_args(self.request.full_url(), page=page + 1)
|
||||
self.set_links(**links)
|
||||
self.write(
|
||||
json.dumps(
|
||||
[
|
||||
|
|
@ -35,7 +72,7 @@ class SearchHandler(tornado.web.RequestHandler):
|
|||
card.color_identity
|
||||
),
|
||||
}
|
||||
for card in cards
|
||||
for card in cards[:limit]
|
||||
]
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@
|
|||
"elm/html": "1.0.0",
|
||||
"elm/http": "2.0.0",
|
||||
"elm/json": "1.1.3",
|
||||
"elm/regex": "1.0.0",
|
||||
"elm/url": "1.0.0",
|
||||
"elm-community/maybe-extra": "5.2.0",
|
||||
"mdgriffith/elm-ui": "1.1.8"
|
||||
},
|
||||
"indirect": {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
<title>Bulk Tagging Dashboard</title>
|
||||
<script type="text/javascript" src="elm.js"></script>
|
||||
</head>
|
||||
<meta charset="utf-8"/>
|
||||
<body>
|
||||
<script type="text/javascript">
|
||||
var app = Elm.App.init({flags: {}});
|
||||
|
|
|
|||
223
www/src/App.elm
223
www/src/App.elm
|
|
@ -1,22 +1,33 @@
|
|||
module App exposing (main)
|
||||
|
||||
import Browser
|
||||
import Browser.Dom
|
||||
import Browser.Events
|
||||
import Dict
|
||||
import Element as E
|
||||
import Element.Background as Background
|
||||
import Element.Border as Border
|
||||
import Element.Font as Font
|
||||
import Element.Input as Input
|
||||
import Html
|
||||
import Html.Events
|
||||
import Http
|
||||
import Json.Decode
|
||||
import Json.Decode.Pipeline as JDP
|
||||
import Maybe.Extra
|
||||
import Regex
|
||||
import Task
|
||||
import Url
|
||||
import Url.Builder
|
||||
|
||||
|
||||
type alias Window =
|
||||
{ width : Int
|
||||
, height : Int
|
||||
}
|
||||
|
||||
|
||||
type alias Criteria =
|
||||
{ name : String
|
||||
{ query : String
|
||||
, ownedOnly : Bool
|
||||
}
|
||||
|
||||
|
|
@ -30,17 +41,20 @@ type alias Card =
|
|||
|
||||
|
||||
type alias Model =
|
||||
{ criteria : Criteria
|
||||
, cards : List Card
|
||||
{ viewport : Window
|
||||
, criteria : Criteria
|
||||
, cardPage : Maybe (ResultPage Card)
|
||||
}
|
||||
|
||||
|
||||
type Msg
|
||||
= UrlChanged Url.Url
|
||||
| ViewportChanged Window
|
||||
| LinkClicked Browser.UrlRequest
|
||||
| UpdateCriteria CriteriaMsg
|
||||
| Search
|
||||
| FoundCards (Result Http.Error (List Card))
|
||||
| GetPage Url.Url
|
||||
| FoundCards (Result Http.Error (ResultPage Card))
|
||||
|
||||
|
||||
type CriteriaMsg
|
||||
|
|
@ -48,13 +62,81 @@ type CriteriaMsg
|
|||
| UpdateOwnedOnly Bool
|
||||
|
||||
|
||||
type alias ResultPage a =
|
||||
{ prev : Maybe Url.Url
|
||||
, next : Maybe Url.Url
|
||||
, values : List a
|
||||
}
|
||||
|
||||
|
||||
expectPaginatedJson : (Result Http.Error (ResultPage value) -> msg) -> Json.Decode.Decoder value -> Http.Expect msg
|
||||
expectPaginatedJson toMsg decoder =
|
||||
let
|
||||
linkPattern : Regex.Regex
|
||||
linkPattern =
|
||||
Regex.fromString "<(.*?)>; rel=\"(.*?)\""
|
||||
|> Maybe.withDefault Regex.never
|
||||
|
||||
links : String -> Dict.Dict String String
|
||||
links s =
|
||||
let
|
||||
toTuples xs =
|
||||
case xs of
|
||||
[ Just a, Just b ] ->
|
||||
Just ( b, a )
|
||||
|
||||
_ ->
|
||||
Nothing
|
||||
in
|
||||
Regex.find linkPattern s
|
||||
|> List.map .submatches
|
||||
|> List.map toTuples
|
||||
|> Maybe.Extra.values
|
||||
|> Dict.fromList
|
||||
in
|
||||
Http.expectStringResponse toMsg <|
|
||||
\response ->
|
||||
case response of
|
||||
Http.BadUrl_ url ->
|
||||
Err (Http.BadUrl url)
|
||||
|
||||
Http.Timeout_ ->
|
||||
Err Http.Timeout
|
||||
|
||||
Http.NetworkError_ ->
|
||||
Err Http.NetworkError
|
||||
|
||||
Http.BadStatus_ metadata _ ->
|
||||
Err (Http.BadStatus metadata.statusCode)
|
||||
|
||||
Http.GoodStatus_ metadata body ->
|
||||
case Json.Decode.decodeString (Json.Decode.list decoder) body of
|
||||
Ok values ->
|
||||
Ok
|
||||
{ prev =
|
||||
Dict.get "link" (Debug.log "headers" metadata.headers)
|
||||
|> Maybe.map links
|
||||
|> Maybe.andThen (Dict.get "prev")
|
||||
|> Maybe.andThen Url.fromString
|
||||
, next =
|
||||
Dict.get "link" metadata.headers
|
||||
|> Maybe.map links
|
||||
|> Maybe.andThen (Dict.get "next")
|
||||
|> Maybe.andThen Url.fromString
|
||||
, values = values
|
||||
}
|
||||
|
||||
Err err ->
|
||||
Err (Http.BadBody (Json.Decode.errorToString err))
|
||||
|
||||
|
||||
search : Criteria -> Cmd Msg
|
||||
search criteria =
|
||||
Http.get
|
||||
{ url =
|
||||
Url.Builder.absolute
|
||||
[ "search" ]
|
||||
[ Url.Builder.string "name" criteria.name
|
||||
[ Url.Builder.string "q" criteria.query
|
||||
, Url.Builder.string "in_collection"
|
||||
(if criteria.ownedOnly then
|
||||
"yes"
|
||||
|
|
@ -62,9 +144,17 @@ search criteria =
|
|||
else
|
||||
""
|
||||
)
|
||||
, Url.Builder.int "limit" 20
|
||||
, Url.Builder.int "limit" 18
|
||||
]
|
||||
, expect = Http.expectJson FoundCards (Json.Decode.list decodeCard)
|
||||
, expect = expectPaginatedJson FoundCards decodeCard
|
||||
}
|
||||
|
||||
|
||||
loadPage : Url.Url -> Cmd Msg
|
||||
loadPage url =
|
||||
Http.get
|
||||
{ url = Url.toString url
|
||||
, expect = expectPaginatedJson FoundCards decodeCard
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -78,29 +168,55 @@ decodeCard =
|
|||
|
||||
|
||||
init : Json.Decode.Value -> url -> key -> ( Model, Cmd Msg )
|
||||
init flag url key =
|
||||
init _ _ _ =
|
||||
let
|
||||
criteria =
|
||||
{ name = "", ownedOnly = True }
|
||||
{ query = "", ownedOnly = True }
|
||||
in
|
||||
( { criteria = criteria
|
||||
, cards = []
|
||||
( { viewport = { width = 1280, height = 720 }
|
||||
, criteria = criteria
|
||||
, cardPage = Nothing
|
||||
}
|
||||
, search criteria
|
||||
, Cmd.batch
|
||||
[ search criteria
|
||||
, Task.perform
|
||||
(\x ->
|
||||
ViewportChanged
|
||||
{ width = floor x.viewport.width
|
||||
, height = floor x.viewport.height
|
||||
}
|
||||
)
|
||||
Browser.Dom.getViewport
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
updateCriteria : CriteriaMsg -> Criteria -> Criteria
|
||||
updateCriteria msg model =
|
||||
case msg of
|
||||
UpdateName text ->
|
||||
{ model | name = text }
|
||||
{ model | query = text }
|
||||
|
||||
UpdateOwnedOnly value ->
|
||||
{ model | ownedOnly = value }
|
||||
|
||||
|
||||
updateCardPage cardpage newCards =
|
||||
{ cardpage | cards = newCards }
|
||||
|
||||
|
||||
update : Msg -> Model -> ( Model, Cmd Msg )
|
||||
update msg model =
|
||||
case msg of
|
||||
UrlChanged _ ->
|
||||
( model, Cmd.none )
|
||||
|
||||
LinkClicked _ ->
|
||||
( model, Cmd.none )
|
||||
|
||||
ViewportChanged viewport ->
|
||||
( { model | viewport = viewport }, Cmd.none )
|
||||
|
||||
UpdateCriteria criteriaMsg ->
|
||||
( { model | criteria = updateCriteria criteriaMsg model.criteria }
|
||||
, Cmd.none
|
||||
|
|
@ -109,10 +225,13 @@ update msg model =
|
|||
Search ->
|
||||
( model, search model.criteria )
|
||||
|
||||
FoundCards (Ok cards) ->
|
||||
( { model | cards = cards }, Cmd.none )
|
||||
GetPage url ->
|
||||
( model, loadPage url )
|
||||
|
||||
_ ->
|
||||
FoundCards (Ok cardPage) ->
|
||||
( { model | cardPage = Just cardPage }, Cmd.none )
|
||||
|
||||
FoundCards (Err _) ->
|
||||
( model, Cmd.none )
|
||||
|
||||
|
||||
|
|
@ -133,19 +252,71 @@ colors =
|
|||
}
|
||||
|
||||
|
||||
viewCard model =
|
||||
viewCardBrowser model =
|
||||
let
|
||||
cardWidth =
|
||||
-- 50% of the Scryfall border_crop image width
|
||||
240
|
||||
|
||||
cardSpacing =
|
||||
10
|
||||
|
||||
navigationButtonWidth =
|
||||
50
|
||||
|
||||
cardColumns =
|
||||
-- Either 3, 6, or 9, based on viewport width
|
||||
let
|
||||
availableColumns =
|
||||
(model.viewport.width - (2 * navigationButtonWidth))
|
||||
// (cardWidth + cardSpacing)
|
||||
in
|
||||
(max 3 >> min 9) (availableColumns // 3 * 3)
|
||||
|
||||
viewCard cardModel =
|
||||
E.column []
|
||||
[ E.image [ E.height <| E.px 300 ]
|
||||
[ E.image [ E.width (E.px cardWidth) ]
|
||||
{ src =
|
||||
Url.Builder.crossOrigin "https://api.scryfall.com"
|
||||
[ "cards", model.scryfallId ]
|
||||
[ "cards", cardModel.scryfallId ]
|
||||
[ Url.Builder.string "format" "image"
|
||||
, Url.Builder.string "version" "border_crop"
|
||||
]
|
||||
, description = model.name
|
||||
, description = cardModel.name
|
||||
}
|
||||
]
|
||||
|
||||
navButton text maybeUrl =
|
||||
case maybeUrl of
|
||||
Just url ->
|
||||
Input.button
|
||||
[ E.height E.fill
|
||||
, E.width (E.px navigationButtonWidth)
|
||||
, Background.color colors.primary
|
||||
, Font.color colors.text
|
||||
, Font.center
|
||||
]
|
||||
{ label = E.text text, onPress = Just (GetPage url) }
|
||||
|
||||
Nothing ->
|
||||
E.el [ E.width (E.px navigationButtonWidth) ] E.none
|
||||
in
|
||||
case model.cardPage of
|
||||
Nothing ->
|
||||
E.none
|
||||
|
||||
Just cardPage ->
|
||||
E.row [ E.width E.fill, E.centerX ] <|
|
||||
[ navButton "←" cardPage.prev
|
||||
, E.wrappedRow
|
||||
[ E.width (E.px (cardColumns * (cardWidth + cardSpacing)))
|
||||
, E.centerX
|
||||
, E.spacing cardSpacing
|
||||
]
|
||||
(List.map viewCard cardPage.values)
|
||||
, navButton "→" cardPage.next
|
||||
]
|
||||
|
||||
|
||||
onEnter : msg -> E.Attribute msg
|
||||
onEnter msg =
|
||||
|
|
@ -168,7 +339,7 @@ view model =
|
|||
{ title = "Tutor"
|
||||
, body =
|
||||
[ E.layout [ Background.color colors.background ] <|
|
||||
E.column []
|
||||
E.column [ E.width E.fill ]
|
||||
[ E.row [ E.spacing 10 ]
|
||||
[ Input.text
|
||||
[ onEnter Search
|
||||
|
|
@ -176,7 +347,7 @@ view model =
|
|||
, Font.color colors.text
|
||||
]
|
||||
{ onChange = UpdateCriteria << UpdateName
|
||||
, text = model.criteria.name
|
||||
, text = model.criteria.query
|
||||
, placeholder = Nothing
|
||||
, label = Input.labelHidden "Search Input"
|
||||
}
|
||||
|
|
@ -196,7 +367,7 @@ view model =
|
|||
, label = Input.labelRight [ Font.color colors.text ] (E.text "Owned only?")
|
||||
}
|
||||
]
|
||||
, E.wrappedRow [ E.spacing 5, E.padding 30 ] <| List.map viewCard model.cards
|
||||
, viewCardBrowser model
|
||||
]
|
||||
]
|
||||
}
|
||||
|
|
@ -212,5 +383,7 @@ main =
|
|||
, subscriptions =
|
||||
\_ ->
|
||||
Sub.batch
|
||||
[]
|
||||
[ Browser.Events.onResize
|
||||
(\w h -> ViewportChanged { width = w, height = h })
|
||||
]
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue