diff --git a/aweber/20210914142000-subscriber_search_service.org b/aweber/20210914142000-subscriber_search_service.org index 64a0468..c09b84a 100644 --- a/aweber/20210914142000-subscriber_search_service.org +++ b/aweber/20210914142000-subscriber_search_service.org @@ -62,6 +62,10 @@ Added: [2019-08-06 Tue 10:34] * Concerns ** Performance +Email delivery has cases + +*** Analytics query speedups for broadcasts +https://jira.aweber.io/browse/EDELIV-8318 ** Fitness to Purpose This service will need to fulfill the needs of both end-user subscriber searches and segment emailing. diff --git a/aweber/20220407112615-null_and_postgresql_array_operators.org b/aweber/20220407112615-null_and_postgresql_array_operators.org new file mode 100644 index 0000000..96d9b80 --- /dev/null +++ b/aweber/20220407112615-null_and_postgresql_array_operators.org @@ -0,0 +1,136 @@ +:PROPERTIES: +:ID: 00cf1628-bc60-451d-bd30-d11d6b92992f +:header-args:sql: :engine postgresql :cmdline "-U postgres postgres" :dir /docker:postgres: :exports both :cache yes :eval no-export +:END: +#+title: Null and PostgreSQL array operators + +Researching an issue in subscriber search and broadcast sending resulted in +learning some interesting things about [[id:af84ed59-96a4-4f9c-b34c-b79178ad20cb][PostgreSQL]]'s generation and handling of +=NULL= values, despite its otherwise strict type checking. + +* The symptom +[[https://jira.aweber.io/browse/ASE-8617][ASE-8617]] describes a scenario where a customer ([[https://admin.aweber.io/account/index/1018872][AID 1018872]]) sends broadcasts to +a segment ("no tags subscribers") on their list (List ID 5830776), defined with +the following criteria: + +#+caption: Segment definition: "no tags subscribers" +| Tag | is not | wr | +| Tag | is not | cb | +| Tag | is not | cb2 | +| Tag | is not | fsc | +| Tag | is not | flp | +| Tag | is not | lsa | + +Normally, they expect this segment to reach >1000 subscribers. Following a +release of our [[id:d06d3ab4-c2d0-47c3-aae1-4395567fc3d2][Tag Normalization]] changes, they sent a broadcast to this segment +that only reached 7 subscribers. Rolling back the [[https://gitlab.aweber.io/DBA/ddl/schema-deploy/-/merge_requests/257][change to segment search terms]] +corrected the issue for the customer. + +* The implementation + +** The "Tag Is Not" search filter +The change updated the tag "is not" filter from +#+begin_src sql :exports code :eval never + (NOT(tags @> ARRAY[=:$1:=]) or tags is null) +#+end_src +to +#+begin_src sql :exports code :eval never + (NOT(public.normalize_tags(tags) @> public.normalize_tags(ARRAY[=:$1:=])) or tags is null) +#+end_src +** The =normalize_tags= function +The =normalize_tags= function is defined as: +#+begin_src sql :exports code :eval never + CREATE OR REPLACE FUNCTION public.normalize_tags(in_tags text[]) + RETURNS text[] + LANGUAGE SQL STRICT IMMUTABLE AS $$ + SELECT ARRAY_AGG(public.normalize_tag(tag)) + FROM UNNEST(in_tags) AS tag; + $$; +#+end_src +* The cause +When testing to see what the criteria SQL would evaluate to, I discovered +something ... interesting. + +#+begin_src sql + SELECT (NOT(public.normalize_tags(ARRAY[]::text[]) @> ARRAY['foo']::text[])) +#+end_src + +#+RESULTS[b0e468c2d8fc47126ee77eb463b2525ff9dca266]: +| ?column? | +|----------| +| | + +That hardly seems right. It's not even returning a boolean. Could it be? + +#+begin_src sql + SELECT (NOT(public.normalize_tags(ARRAY[]::text[]) @> ARRAY['foo']::text[])) IS NULL +#+end_src + +#+RESULTS[c30ad761b00b6d3c038ec008010f17ae5228befd]: +| ?column? | +|----------| +| t | + +Yep. It's returning =NULL=, Tony Hoare's "[[https://www.infoq.com/presentations/Null-References-The-Billion-Dollar-Mistake-Tony-Hoare/][billion-dollar mistake]]". The only +thing different in this implementation is the call to =normalize_tags=, so let's +take a look at that. + +#+begin_src sql + SELECT public.normalize_tags(ARRAY[]::text[]) IS NULL +#+end_src + +#+RESULTS[8a85412be393d1e19d576db8ef321f0ab1380261]: +| ?column? | +|----------| +| t | + +Sure enough, it is returning =NULL= when called with an empty array. That =NULL= then escapes to our array comparison... + +#+begin_src sql + SELECT (NULL @> ARRAY['foo'::text]) IS NULL +#+end_src + +#+RESULTS[80d59a923b98d6ec7adc7af29a53b7557c7c0efc]: +| ?column? | +|----------| +| t | + +... which also returns =NULL=. That then gets passed to =NOT=... + +#+begin_src sql + SELECT (NOT(NULL)) IS NULL +#+end_src + +#+RESULTS[77051c05ab0f6df48e3c13be6ff0651904f8b6da]: +| ?column? | +|----------| +| t | + +... which also returns =NULL=. =NULL= is treated as falsy by [[id:af84ed59-96a4-4f9c-b34c-b79178ad20cb][PostgreSQL]], +and so the search fails to match subscribers without tags. +* The resolution +Using =COALESCE= on the result of =ARRAY_AGG= to ensure we get an empty array +when the result is =NULL=, which lets us avoid all of the above problems caused +by =NULL= escaping into our comparisons. +#+begin_src sql :exports code :eval no-export + CREATE OR REPLACE FUNCTION public.normalize_tags(in_tags text[]) + RETURNS text[] + LANGUAGE SQL STRICT IMMUTABLE AS $$ + SELECT COALESCE(ARRAY_AGG(public.normalize_tag(tag)), ARRAY[]::text[]) + FROM UNNEST(in_tags) AS tag + WHERE public.normalize_tag(tag) <> ''; + $$; +#+end_src + +#+RESULTS[f49761b0fa0e3074cf739102361da7272672f0a1]: +| CREATE FUNCTION | +|-----------------| + +#+begin_src sql + SELECT public.normalize_tags(ARRAY[]::text[]) +#+end_src + +#+RESULTS[a37fd5ae0eaedaf2b33b4886d60d2db0cf632fb5]: +| normalize_tags | +|----------------| +| {} | diff --git a/daily/2022-03-22.org b/daily/2022-03-22.org index 520bc97..2061e3f 100644 --- a/daily/2022-03-22.org +++ b/daily/2022-03-22.org @@ -214,3 +214,130 @@ if the event being compared is a =tag.v1= event. Updated =rulesengine= in https://gitlab.aweber.io/CC/Libraries/rulesengine/-/merge_requests/117. + +** Looking up a campaign's ruleset +Fetch the campaign record using the UUID in the control0-panel campaign URL. +#+begin_src http :pretty :cache yes :exports both + GET http://campaign.service.staging.consul/210b4a4f-ee07-4f1e-98c8-d16541f6e686 +#+end_src + +#+RESULTS[c237f01dc854e750aa1c8f568dbd54f77ab6f56c]: +#+begin_example +{ + "id": "210b4a4f-ee07-4f1e-98c8-d16541f6e686", + "name": "Normal Scott", + "owner": "26b49bc2-207e-4a33-b267-5b9e64f2702e", + "parent": "9f1db623-fbc3-4112-a2f3-ab563e37e131", + "rule_set": "01955ad5-cbca-455c-934f-d69a71a5578f", + "created_at": "2022-03-21T20:17:39.795431+00:00", + "modified_at": "2022-03-24T18:34:37.909620+00:00", + "state_changed_at": "2022-03-22T15:24:45.727430+00:00", + "url": "http://campaign.service.staging.consul/210b4a4f-ee07-4f1e-98c8-d16541f6e686?owner=26b49bc2-207e-4a33-b267-5b9e64f2702e&parent=9f1db623-fbc3-4112-a2f3-ab563e37e131", + "state": "active", + "state_serial": 3, + "is_valid": true, + "is_legacy_followup": false, + "sharing_enabled": false, + "auto_extend_enabled": true, + "extend_needed": false, + "extended_at": null, + "custom_fields": [], + "global_fields": [], + "campaign_type": "autoresponder", + "properties": null +} +#+end_example + +Fetch the rule set using the =rule_set= id returned with the campaign. + +#+begin_src http :pretty :cache yes :exports both + GET https://rule.aweberstage.com/01955ad5-cbca-455c-934f-d69a71a5578f +#+end_src + +#+RESULTS: +#+begin_example +{ + "id": "01955ad5-cbca-455c-934f-d69a71a5578f", + "version": 2, + "owner": "26b49bc2-207e-4a33-b267-5b9e64f2702e", + "parent": "9f1db623-fbc3-4112-a2f3-ab563e37e131", + "state": "active", + "system": false, + "iterator": null, + "events": [ + { + "id": "b3b69bd4-0c7b-4253-ab65-e6fcbb726f07", + "type": "tag.v1", + "title": null, + "filter": { + "type": "all", + "criteria": [ + { + "expect": true, + "kwargs": { + "key": "label", + "values": [ + "normalize test", + "foo" + ] + }, + "function": "rulesengine.filter.event_value_in", + "operator": "==" + }, + { + "expect": true, + "kwargs": { + "list": "", + "labels": [ + "normalize test", + "foo" + ], + "account": "", + "recipient": "" + }, + "function": "ruleset.tag.filter.any_tags_v1", + "operator": "==" + }, + { + "expect": false, + "kwargs": { + "list": "", + "labels": [ + "normalized tag" + ], + "account": "", + "recipient": "" + }, + "function": "ruleset.tag.filter.any_tags_v1", + "operator": "==" + } + ] + }, + "parents": [], + "metadata": "tag.v1", + "recurring": false + } + ], + "actions": [ + { + "id": "396cd5d3-d410-4ae2-a991-b93a0b816600", + "title": null, + "parents": [ + "b3b69bd4-0c7b-4253-ab65-e6fcbb726f07" + ], + "metadata": "send-message", + "recurring": false, + "definition": { + "kwargs": { + "list": "", + "account": "", + "message": "394c27bf-5853-4b45-9ff1-a0637848f4b6", + "meapi_id": "6238dd85f227ef3f2d5ec3a5", + "recipient": "" + }, + "function": "ruleset.email.action.compose_v1" + } + } + ] +} +#+end_example diff --git a/daily/2022-03-29-accounts-with-invalid-tags.png b/daily/2022-03-29-accounts-with-invalid-tags.png new file mode 100644 index 0000000..15a8243 Binary files /dev/null and b/daily/2022-03-29-accounts-with-invalid-tags.png differ diff --git a/daily/2022-03-29-invalid-tag-breakdown.png b/daily/2022-03-29-invalid-tag-breakdown.png new file mode 100644 index 0000000..8954966 Binary files /dev/null and b/daily/2022-03-29-invalid-tag-breakdown.png differ diff --git a/daily/2022-03-29-subscribers-with-invalid-tags.png b/daily/2022-03-29-subscribers-with-invalid-tags.png new file mode 100644 index 0000000..e235209 Binary files /dev/null and b/daily/2022-03-29-subscribers-with-invalid-tags.png differ diff --git a/daily/2022-03-29.org b/daily/2022-03-29.org index 9317f01..59ceb39 100644 --- a/daily/2022-03-29.org +++ b/daily/2022-03-29.org @@ -28,56 +28,200 @@ rules. WHERE tag != normalize_tag(tag) #+end_src -** Subscribers on active accounts -#+begin_src sql - SELECT COUNT(*) FROM subscribers +** Active accounts +#+name: total-accounts +#+begin_src sql :post humanize-numbers-in-results(results=*this*) + SELECT COUNT(DISTINCT account_id) FROM subscribers #+end_src -#+RESULTS[073c7d5b524a0577c83785ca6051ad010990c18a]: -| count | -|----------| -| 16650026 | +#+RESULTS[5d46a920e8c34c7fd755c2d1fd82ba567f8341d9]: total-accounts +| count | +|---------| +| 103,357 | + +** Subscribers on active accounts +#+name: total-subscribers +#+begin_src sql :post humanize-numbers-in-results(results=*this*) + SELECT COUNT(id) FROM subscribers +#+end_src + +#+RESULTS[17e3b468a66f03b74bdddd77044359dd8e4e92e3]: total-subscribers +| count | +|-------------| +| 259,745,858 | ** Subscribers with invalid tags -#+begin_src sql +#+name: subscribers-with-invalid-tags +#+begin_src sql :post humanize-numbers-in-results(results=*this*) SELECT COUNT(DISTINCT subscriber_id) FROM invalid_tags #+end_src -#+RESULTS[6ff96db184978da7271a89499c58d40cff8daebf]: -| count | -|--------| -| 239291 | +#+RESULTS[b1e6beccd5a3bd4e503473844cb12e6d03d2c21d]: subscribers-with-invalid-tags +| count | +|-----------| +| 1,331,220 | + +#+RESULTS: +#+begin_src emacs-lisp +"259,745,858" +#+end_src + +#+header: :var filename="2022-03-29-subscribers-with-invalid-tags.png" +#+header: :var total=total-subscribers[2,0] +#+header: :var affected=subscribers-with-invalid-tags[2,0] +#+begin_src python :exports results :results file + import matplotlib.pyplot as plt + + plt.style.use("seaborn-pastel") + + total = int(str(total).replace(',', '')) + affected = int(str(affected).replace(',', '')) + + fig1, ax1 = plt.subplots() + ax1.pie( + [affected, total - affected], + explode=[0.5, 0], + labels=[f"Affected", "Unaffected"], + autopct="%1.1f%%", + ) + plt.savefig(filename) + return filename +#+end_src + +#+RESULTS: +[[file:2022-03-29-subscribers-with-invalid-tags.png]] ** Accounts with subscribers with invalid tags -#+begin_src sql +#+name: accounts-with-invalid-tags +#+begin_src sql :post humanize-numbers-in-results(results=*this*) SELECT COUNT(DISTINCT account_id) FROM invalid_tags; #+end_src -#+RESULTS[f45bb88f747f30295132bcd75861b98721cd5341]: +#+RESULTS[0ee163f090d0e32e6a45b5b859948882ffe021d0]: accounts-with-invalid-tags | count | |-------| -| 16 | +| 3,220 | -#+begin_src sql - SELECT DISTINCT account_id FROM invalid_tags +#+header: :var filename="2022-03-29-accounts-with-invalid-tags.png" +#+header: :var total=total-accounts[2,0] +#+header: :var affected=accounts-with-invalid-tags[2,0] +#+begin_src python :exports results :results file + import matplotlib.pyplot as plt + + plt.style.use("seaborn-pastel") + + total = int(str(total).replace(',', '')) + affected = int(str(affected).replace(',', '')) + + fig1, ax1 = plt.subplots() + ax1.pie( + [affected, total - affected], + explode=[0.5, 0], + labels=[f"Affected", "Unaffected"], + autopct="%1.1f%%", + ) + plt.savefig(filename) + return filename #+end_src -#+RESULTS[ff2ba7ea0c8dc285f2be2a20afb0d0c786f4dac1]: -| account_id | -|------------| -| 1833 | -| 211344 | -| 13492 | -| 212837 | -| 216738 | -| 213819 | -| 16479 | -| 215217 | -| 104120 | -| 215067 | -| 213122 | -| 14656 | -| 111262 | -| 214928 | -| 44749 | -| 91 | +#+RESULTS: +[[file:2022-03-29-accounts-with-invalid-tags.png]] + +** Normalized tag breakdown +#+name: tag-breakdown +#+begin_src sql :post humanize-numbers-in-results(results=*this*) + SELECT 'Non-printable characters' AS "Rule" + , COUNT(DISTINCT account_id) AS "Accounts" + , COUNT(subscriber_id) AS "Subscribers" + FROM invalid_tags + WHERE tag ~ '[^[:print:]]' + UNION SELECT 'Commas' AS "Rule" + , COUNT(DISTINCT account_id) AS "Accounts" + , COUNT(subscriber_id) AS "Subscribers" + FROM invalid_tags + WHERE tag ~ ',' + UNION SELECT 'ASCII quotation marks' AS "Rule" + , COUNT(DISTINCT account_id) AS "Accounts" + , COUNT(subscriber_id) AS "Subscribers" + FROM invalid_tags + WHERE tag ~ '[''""]' + UNION SELECT 'Unicode quotation marks' AS "Rule" + , COUNT(DISTINCT account_id) AS "Accounts" + , COUNT(subscriber_id) AS "Subscribers" + FROM invalid_tags + WHERE tag ~ '[‘’“”]' + UNION SELECT 'Leading or trailing whitespace' AS "Rule" + , COUNT(DISTINCT account_id) AS "Accounts" + , COUNT(subscriber_id) AS "Subscribers" + FROM invalid_tags + WHERE TRIM(tag) != tag + UNION SELECT 'Repeated whitespace' AS "Rule" + , COUNT(DISTINCT account_id) AS "Accounts" + , COUNT(subscriber_id) AS "Subscribers" + FROM invalid_tags + WHERE TRIM(tag) ~ '[:space:]{2,}' + UNION SELECT 'Upper-case characters' AS "Rule" + , COUNT(DISTINCT account_id) AS "Accounts" + , COUNT(subscriber_id) AS "Subscribers" + FROM invalid_tags + WHERE LOWER(tag) != tag +#+end_src + +#+RESULTS[853c04719cf6fd9c329359c1b72e8198393322b9]: tag-breakdown +| Rule | Accounts | Subscribers | +|--------------------------------+----------+-------------| +| Leading or trailing whitespace | 119 | 66,788 | +| Repeated whitespace | 2,404 | 1,234,651 | +| Unicode quotation marks | 126 | 21,343 | +| Commas | 378 | 54,567 | +| ASCII quotation marks | 2,507 | 1,544,607 | +| Upper-case characters | 0 | 0 | +| Non-printable characters | 58 | 1,749 | + +#+header: :var filename="2022-03-29-invalid-tag-breakdown.png" +#+header: :var breakdown=tag-breakdown +#+begin_src python :exports results :results file + import matplotlib.pyplot as plt + + plt.style.use("seaborn-pastel") + + accounts = sorted( + [(int(str(row[1]).replace(",", "")), row[0]) for row in breakdown], + reverse=True, + ) + account_total = sum(a[0] for a in accounts) + subscribers = sorted( + [(int(str(row[2]).replace(",", "")), row[0]) for row in breakdown], + reverse=True, + ) + subscriber_total = sum(s[0] for s in subscribers) + + fig1, axs = plt.subplots(2, 2) + axs[0, 0].set_title("Accounts") + wedges1, _ = axs[0, 0].pie( + [a[0] for a in accounts], + explode=[0.1 for _ in accounts], + ) + axs[1, 0].axis("off") + axs[1, 0].legend( + wedges1, + ["{:.1f}% {}".format(a[0] * 100.0 / account_total, a[1]) for a in accounts], + fontsize=8, + ) + axs[0, 1].set_title("Subscribers") + wedges2, _ = axs[0, 1].pie( + [s[0] for s in subscribers], + explode=[0.1 for _ in subscribers], + ) + axs[1, 1].axis("off") + axs[1, 1].legend( + wedges2, + ["{:.1f}% {}".format(s[0] * 100.0 / subscriber_total, s[1]) for s in subscribers], + fontsize=8, + ) + plt.savefig(filename) + return filename +#+end_src + +#+RESULTS: +[[file:2022-03-29-invalid-tag-breakdown.png]] diff --git a/daily/2022-04-06.org b/daily/2022-04-06.org new file mode 100644 index 0000000..ef116cc --- /dev/null +++ b/daily/2022-04-06.org @@ -0,0 +1,27 @@ +:PROPERTIES: +:ID: 091bce6d-e15b-434f-a7be-bc6627a71925 +:END: +#+title: 2022-04-06 +* Fixing normalized tag search +https://jira.aweber.io/browse/ASE-8617 + +- *What was broken?* :: Searches and segments with a "tag is not" constraint + failed to match subscribers with no tags. +- *When was it broken?* :: April 5th at 16:00, following the release of + CCPANEL-12034. +- *What did you do to fix the problem?* :: The change was rolled back by + updating search terms in the analytics search database to their previous + values at 19:12 on April 5th. +- *How many customers did it likely impact?* :: Likely all customers using the + "tag not in" filter in their searches and segments. +- *Is the issue automatically fixed for all customers now?* :: Yes. +- *Does the customer or CS need to manually do something to fix their account?* :: No. +- *Should a new monitoring check, metric, or test be created to prevent this from happening again?* :: Additional + tests will be added to ensure future releases of this feature do not break + search and segment behavior. +** What happened? +=ARRAY_AGG= returns =NULL= when given an empty set. This breaks the tag +comparison, as =NOT(NULL @> ARRAY['tag'])= is equal to =NULL=, not =TRUE=. +Updating the logic to =COALESCE= null values with an empty array allows the +comparison to function as intended. =normalize_tags= must also omit invalid tags +(tags that normalize to the empty string) from its results. diff --git a/daily/2022-04-07.org b/daily/2022-04-07.org new file mode 100644 index 0000000..f8aaf26 --- /dev/null +++ b/daily/2022-04-07.org @@ -0,0 +1,92 @@ +:PROPERTIES: +:ID: 5762836c-ef1f-43f5-9562-3bc38ae411ed +:END: +#+title: 2022-04-07 +* Verifying search tag normalization changes +:PROPERTIES: +:header-args:sql: :engine postgresql :cmdline "-U postgres postgres" :dir /docker:postgres: :exports both :cache yes +:END: +- https://jira.aweber.io/browse/ASE-8617 +- https://admin.aweber.io/account/index/1018872 +- List 5830776 + +#+caption: Segment definition: "no tags subscribers" + | Tag | is not | wr | + | Tag | is not | cb | + | Tag | is not | cb2 | + | Tag | is not | fsc | + | Tag | is not | flp | + | Tag | is not | lsa | +** Using the old =normalize_tags= implementation +#+begin_src sql :eval never + CREATE OR REPLACE FUNCTION public.normalize_tags(in_tags text[]) + RETURNS text[] + LANGUAGE SQL STRICT IMMUTABLE AS $$ + SELECT ARRAY_AGG(public.normalize_tag(tag)) + FROM UNNEST(in_tags) AS tag; + $$; + + SELECT COUNT(t.subscriber_id) + FROM subscriber_tags AS t + JOIN subscribers AS s ON (s.id = t.subscriber_id) + WHERE s.account_id = 1018872 AND s.list_id = 5830776 + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['wr'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['cb'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['cb2'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['fsc'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['flp'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['lsa'::text])); +#+end_src + +#+RESULTS[d56d997121fe4f3ff98458e18fba912650d44796]: +| CREATE FUNCTION | +|-----------------| +| count | +| 0 | + +** Using the old comparisons +#+begin_src sql :eval never + SELECT COUNT(t.subscriber_id) + FROM subscriber_tags AS t + JOIN subscribers AS s ON (s.id = t.subscriber_id) + WHERE s.account_id = 1018872 AND s.list_id = 5830776 + AND NOT(t.tags @> ARRAY['wr'::text]) + AND NOT(t.tags @> ARRAY['cb'::text]) + AND NOT(t.tags @> ARRAY['cb2'::text]) + AND NOT(t.tags @> ARRAY['fsc'::text]) + AND NOT(t.tags @> ARRAY['flp'::text]) + AND NOT(t.tags @> ARRAY['lsa'::text]); +#+end_src + +#+RESULTS[792264b3be7d158bb6bda1124a4e23f4a19ada06]: +| count | +|-------| +| 1016 | + +** Using the new =normalize_tags= implementation +#+begin_src sql :eval never + CREATE OR REPLACE FUNCTION public.normalize_tags(in_tags text[]) + RETURNS text[] + LANGUAGE SQL STRICT IMMUTABLE AS $$ + SELECT COALESCE(ARRAY_AGG(public.normalize_tag(tag)), ARRAY[]::text[]) + FROM UNNEST(in_tags) AS tag + WHERE public.normalize_tag(tag) <> ''; + $$; + + SELECT COUNT(t.subscriber_id) + FROM subscriber_tags AS t + JOIN subscribers AS s ON (s.id = t.subscriber_id) + WHERE s.account_id = 1018872 AND s.list_id = 5830776 + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['wr'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['cb'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['cb2'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['fsc'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['flp'::text])) + AND NOT(public.normalize_tags(t.tags) @> public.normalize_tags(ARRAY['lsa'::text])); +#+end_src + +#+RESULTS[ae72571ff73254fc70555db4517f759ddfdfcc8c]: +| CREATE FUNCTION | +|-----------------| +| count | +| 1016 | diff --git a/daily/2022-04-12.org b/daily/2022-04-12.org new file mode 100644 index 0000000..1f49ca1 --- /dev/null +++ b/daily/2022-04-12.org @@ -0,0 +1,144 @@ +:PROPERTIES: +:ID: b1d8fa03-4bd6-4bae-a642-23ae77769c41 +:END: +#+title: 2022-04-12 +* Normalizing tags in rule sets +References for applying [[id:d06d3ab4-c2d0-47c3-aae1-4395567fc3d2][Tag Normalization]] to campaign rule sets. + +Campaign proxy iterates through this data, could be useful as an example. +** References for navigating rule set data structures +- Ruleset structure :: https://confluence.aweber.io/display/AR/Ruleset+v2+Structure#Rulesetv2Structure-event +- Ruleset json schemas :: https://gitlab.aweber.io/CC/Libraries/ruleset-json-schema/-/tree/master +- Campaigns iterating through events :: https://gitlab.aweber.io/CC/Applications/campaign-proxy/-/blob/4eafb44c557aff34a0c9b519ff99d6c81eb9b0d8/campaignproxy/ruleset_validation_handlers.py#L442-447 +- Campaigns iterating through actions :: https://gitlab.aweber.io/CC/Applications/campaign-proxy/-/blob/4eafb44c557aff34a0c9b519ff99d6c81eb9b0d8/campaignproxy/ruleset_validation_handlers.py#L402-403 + +#+begin_notes +The rule service also stores data for searches in redis, be sure to alter the +data before any storage occurs. +#+end_notes +** Looking up a campaign's ruleset +Fetch the campaign record using the UUID in the control0-panel campaign URL. +#+begin_src http :pretty :cache yes :exports both + GET http://campaign.service.staging.consul/210b4a4f-ee07-4f1e-98c8-d16541f6e686 +#+end_src + +#+RESULTS[c237f01dc854e750aa1c8f568dbd54f77ab6f56c]: +#+begin_example +{ + "id": "210b4a4f-ee07-4f1e-98c8-d16541f6e686", + "name": "Normal Scott", + "owner": "26b49bc2-207e-4a33-b267-5b9e64f2702e", + "parent": "9f1db623-fbc3-4112-a2f3-ab563e37e131", + "rule_set": "01955ad5-cbca-455c-934f-d69a71a5578f", + "created_at": "2022-03-21T20:17:39.795431+00:00", + "modified_at": "2022-03-24T18:34:37.909620+00:00", + "state_changed_at": "2022-03-22T15:24:45.727430+00:00", + "url": "http://campaign.service.staging.consul/210b4a4f-ee07-4f1e-98c8-d16541f6e686?owner=26b49bc2-207e-4a33-b267-5b9e64f2702e&parent=9f1db623-fbc3-4112-a2f3-ab563e37e131", + "state": "active", + "state_serial": 3, + "is_valid": true, + "is_legacy_followup": false, + "sharing_enabled": false, + "auto_extend_enabled": true, + "extend_needed": false, + "extended_at": null, + "custom_fields": [], + "global_fields": [], + "campaign_type": "autoresponder", + "properties": null +} +#+end_example + +Fetch the rule set using the =rule_set= id returned with the campaign. + +#+begin_src http :pretty :cache yes :exports both + GET https://rule.aweberstage.com/01955ad5-cbca-455c-934f-d69a71a5578f +#+end_src + +#+RESULTS[685774a46bbaf38b945afcc0952a390d1bc53ed3]: +#+begin_example +{ + "id": "01955ad5-cbca-455c-934f-d69a71a5578f", + "version": 2, + "owner": "26b49bc2-207e-4a33-b267-5b9e64f2702e", + "parent": "9f1db623-fbc3-4112-a2f3-ab563e37e131", + "state": "active", + "system": false, + "iterator": null, + "events": [ + { + "id": "b3b69bd4-0c7b-4253-ab65-e6fcbb726f07", + "type": "tag.v1", + "title": null, + "filter": { + "type": "all", + "criteria": [ + { + "expect": true, + "kwargs": { + "key": "label", + "values": [ + "normalize test", + "foo" + ] + }, + "function": "rulesengine.filter.event_value_in", + "operator": "==" + }, + { + "expect": true, + "kwargs": { + "list": "", + "labels": [ + "normalize test", + "foo" + ], + "account": "", + "recipient": "" + }, + "function": "ruleset.tag.filter.any_tags_v1", + "operator": "==" + }, + { + "expect": false, + "kwargs": { + "list": "", + "labels": [ + "normalized tag" + ], + "account": "", + "recipient": "" + }, + "function": "ruleset.tag.filter.any_tags_v1", + "operator": "==" + } + ] + }, + "parents": [], + "metadata": "tag.v1", + "recurring": false + } + ], + "actions": [ + { + "id": "396cd5d3-d410-4ae2-a991-b93a0b816600", + "title": null, + "parents": [ + "b3b69bd4-0c7b-4253-ab65-e6fcbb726f07" + ], + "metadata": "send-message", + "recurring": false, + "definition": { + "kwargs": { + "list": "", + "account": "", + "message": "394c27bf-5853-4b45-9ff1-a0637848f4b6", + "meapi_id": "6238dd85f227ef3f2d5ec3a5", + "recipient": "" + }, + "function": "ruleset.email.action.compose_v1" + } + } + ] +} +#+end_example