:PROPERTIES: :ID: fce5a567-37a4-414e-bb50-5ffb35d6b43b :END: #+title: 2022-05-11 * Normalizing stored tag data :PROPERTIES: :header-args:sql: :eval never :header-args:http: :eval never :END: Preparing to run data updates for [[id:d06d3ab4-c2d0-47c3-aae1-4395567fc3d2][Tag Normalization]]. ** Backing up existing data *** Subscriber Tags # #+header: :dbhost app.service.staging.consul :dbport 6000 :database app # #+header: :dbpassword (password-store-get "Work/aweber/aweber.io/correlr") #+header: :var account=778 #+begin_src sql :engine postgresql SELECT s.*, t.* -- FROM public.subscriber_tags AS t JOIN list.subscribers AS s ON (s.id = t.subscriber_id) WHERE s.account_id = $account LIMIT 3 #+end_src #+RESULTS: | id | list_id | account_id | lead_id | leads_table | subscriber_id | version | tags | |------------+---------+------------+---------+-------------+---------------+---------+--------------| | 1224565879 | 3854 | 778 | 4496733 | leads4 | 1224565879 | 1 | {deleteme} | | 1224565867 | 3854 | 778 | 4496722 | leads4 | 1224565867 | 1 | {deletetest} | | 1224515286 | 3854 | 778 | 4472377 | leads4 | 1224515286 | 2 | {} | *** Segments # #+header: :dbhost (my/consul-kv "services/cp/applications/control-panel/search_master/hostname" "staging") # #+header: :dbuser (my/consul-kv "services/cp/applications/control-panel/search_master/username" "staging") # #+header: :database (my/consul-kv "services/cp/applications/control-panel/search_master/dbname" "staging") # #+header: :dbpassword (my/consul-kv "services/cp/applications/control-panel/search_master/password" "staging") #+header: :var account=778 #+begin_src sql :engine postgresql SELECT c.* FROM lead_views AS v JOIN lead_view_criteria as c ON (c.lead_view_id = v.id) WHERE v.a_id = $account #+end_src #+RESULTS: | id | search_box_id | lead_view_id | criteria | |-------+---------------+--------------+-------------------------------------------| | 40446 | 659 | 54648 | "barfarts" | | 40447 | 660 | 54649 | "barfarts" | | 40448 | 659 | 54650 | scott miller | | 40449 | 660 | 54651 | scott miller | | 40450 | 659 | 54652 | scott miller | | 40451 | 660 | 54653 | scott miller | | 26538 | 11 | 42564 | Scott | | 26329 | 177 | 42351 | 1243650 | | 40462 | 666 | 54660 | {"normalized tag"} | | 40463 | 666 | 54661 | {"normalized' tag"} | | 40464 | 667 | 54662 | {"normalized tag","normalized' tag"} | *** Rulesets ** Choosing accounts to run against #+begin_src sql :eval never SELECT distinct account_id, count(subscriber_id) FROM list.subscribers AS ls JOIN public.subscriber_tags st ON (ls.id = st.subscriber_id) WHERE tags <> NORMALIZE_TAGS(st.tags) GROUP BY account_id; #+end_src | account_id | count | |------------+-------| | 420 | 83 | | 734 | 57 | | 386 | 13 | | 778 | 11 | | 387 | 11 | | 399 | 6 | | 817965 | 5 | | 14297 | 5 | | 396 | 2 | | 295 | 2 | | 914014 | 1 | | 14435 | 1 | | 614 | 1 | | 992770 | 1 | | 91 | 1 | #+begin_src shell :eval never psql -h app.service.staging.consul -p 6000 -d app -c 'COPY (SELECT t.* FROM public.subscriber_tags AS t JOIN list.subscribers AS s ON (s.id = t.subscriber_id) WHERE s.account_id = 420) TO STDOUT WITH CSV HEADER' 1> subscriber_tags.staging.420.`date +%Y%m%d-%H%M%S`.csv #+end_src * One of Ihar's subscribers has a new UUID? #+begin_src http GET http://mapping.aweberstage.com/f1b9e421-09e4-4b83-9150-b1e3cc7ea200 #+end_src #+RESULTS: #+begin_example HTTP/1.1 200 OK Date: Wed, 11 May 2022 18:58:00 GMT Content-Type: application/json; charset="utf-8" Content-Length: 89 Connection: keep-alive Server: mapping/2.2.5 Correlation-Id: 853b63a6-677f-454e-8a43-d80ecda5a685 Cache-Control: public, max-age=2592000 Vary: Accept,Accept, Accept-Encoding Last-Modified: Mon, 31 Jan 2022 20:17:45 +0000 {"id": "f1b9e421-09e4-4b83-9150-b1e3cc7ea200", "type": "subscriber", "value": 1224242413} #+end_example #+begin_src http GET http://mapping.aweberstage.com/d0ef92be-0330-41a5-ba75-d7ef25e43fd8 #+end_src #+RESULTS: #+begin_example HTTP/1.1 200 OK Date: Wed, 11 May 2022 18:58:24 GMT Content-Type: application/json; charset="utf-8" Content-Length: 89 Connection: keep-alive Server: mapping/2.2.5 Correlation-Id: 3f0c4cff-6574-4867-9896-8966762d6da8 Cache-Control: public, max-age=2592000 Vary: Accept,Accept, Accept-Encoding Last-Modified: Mon, 02 May 2022 22:12:29 +0000 {"id": "d0ef92be-0330-41a5-ba75-d7ef25e43fd8", "type": "subscriber", "value": 1224589131} #+end_example #+begin_src http GET http://subscriber.service.staging.consul/subscriber/1224242413 #+end_src #+RESULTS: : HTTP/1.1 404 SubscriberNotFound : Server: nginx : Date: Wed, 11 May 2022 18:59:37 GMT : Content-Type: application/json; charset=UTF-8 : Content-Length: 126 : Connection: keep-alive : Correlation-Id: 228f6aa9-03b5-4e7e-9663-c5d492c12d95 : : {"error": {"status_code": 404, "exception": "SubscriberNotFound", "message": "Could not find information for the subscriber"}} #+begin_src http :pretty GET http://subscriber.service.staging.consul/subscriber/1224589131 #+end_src #+RESULTS: #+begin_example { "isp": null, "subscriber_id": 1224589131, "legacy_name": "", "subscriber_source": "control panel", "dma_code": "504", "id": 1224589131, "custom_fields": {}, "subscriber_uuid": "d0ef92be-0330-41a5-ba75-d7ef25e43fd8", "city": "Chalfont", "verified": 0, "self": "http://subscriber.service.staging.consul/subscriber/1224589131", "verification_time": null, "subscribe_method": "webform", "stop_time": null, "list_id": 27330, "latitude": 40.2874, "email": "campaign_regression_tag.se1a93xt@mailosaur.io", "status": "subscribed", "last_followup": 0, "area_code": null, "followuptime": null, "org": null, "postal": "18914", "ip_address": "204.194.222.13", "name": "Campaigns Regression Tag", "lead_id": 4510924, "add_url": "https://www.aweberstage.com/users/leads/add", "country": "United States", "region": "PA", "unsubscribe_source": null, "longitude": -75.2156, "subscribed_at": "2022-05-02T18:12:28-04:00", "stop_method": null, "unsubscribe_timestamp": null } #+end_example